diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ade76452..320e1e82 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -53,6 +53,7 @@ jobs: from pathlib import Path import dataretrieval + import dataretrieval.transport from dataretrieval import ngwmn, waterdata, wateruse from dataretrieval.ogc import engine @@ -60,7 +61,15 @@ jobs: installed = Path(dataretrieval.__file__).resolve() assert not installed.is_relative_to(checkout), (installed, checkout) assert importlib.util.find_spec("dataretrieval.waterdata.api") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.time_series") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.metadata") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.measurements") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.reference") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.samples") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.cql") is not None assert importlib.util.find_spec("dataretrieval.ogc.engine") is not None + assert importlib.util.find_spec("dataretrieval.ogc.context") is not None + assert importlib.util.find_spec("dataretrieval.ogc.schema") is not None assert files("dataretrieval").joinpath("py.typed").is_file() assert waterdata.get_daily assert ngwmn.get_sites diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 024d6192..f8667780 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,6 +117,9 @@ link checking. * Group public download functions by data portal. For example, modern Water Data functions belong in `dataretrieval.waterdata`; legacy NWIS functions remain quarantined in `dataretrieval.nwis` during deprecation. +* Treat a change to a service's documented return shape or metadata type as a + public compatibility change; update contract tests and architecture + documentation and follow the deprecation process where required. * Preserve the dependency direction documented in [`docs/source/architecture`](docs/source/architecture/index.rst): public facades depend on service/protocol adapters, which depend on stable shared diff --git a/NEWS.md b/NEWS.md index 861d2c5c..7c926476 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +**08/03/2026:** Split the Water Data implementation into focused time-series, metadata, measurements, reference, samples, and CQL collection-family modules behind the unchanged `waterdata.api` facade. Active service modules now declare explicit exports; public Water Data imports, signatures, function identities, deprecations, and return contracts are protected by executable contract snapshots. OGC ambient context and schema/queryables execution are separated from request construction, adapter-to-adapter reach-through is prohibited by architecture tests, and service-specific output shapes are documented rather than forced into one model. + +**08/02/2026:** Added an internal API-neutral transport layer for guarded HTTP clients, host-scoped authentication, cursor pagination, bounded retry, response aggregation, progress, and sync-over-async dispatch. Water Use and the non-OGC Statistics API now consume transport directly instead of private OGC execution helpers; WQP, NLDI, and StreamStats opt into bounded transient retry while deprecated NWIS behavior remains unchanged. OGC retains CQL2, request construction, feature shaping, chunk planning, resumable calls, and interruption types, with compatibility imports at previous private paths. Failed pagination and fan-out still raise rather than returning partial data, and no public signatures or return shapes changed. + **08/02/2026:** Phase 1 OGC boundary stabilization: the `dataretrieval.ogc` package now exposes a deliberate, small facade (`OgcDialect`, `prepare_request_args`, `get_ogc_data`, `fetch_ogc_request`) that service adapters (NGWMN, Water Data's generic wrapper) import. Internal request-construction helpers moved to a new `ogc.requests` module; the dialect type and endpoint constants live in the leaf `ogc.policy` module. `ogc.shaping` no longer depends on `ogc.engine` at all, and the complete runtime OGC import graph is now acyclic. `_default_headers` now accepts a target URL and adds `X-Api-Key` only for `api.waterdata.usgs.gov`; shared sync and async clients also strip the key before following any cross-host redirect, including external rating-asset downloads. `waterdata.utils` no longer bulk-re-exports private OGC symbols, and consumers import implementation helpers from their canonical modules. No public API, return-value, or deprecation changes. **08/02/2026:** Fixed source-distribution and wheel package discovery so the `dataretrieval.ogc` and `dataretrieval.waterdata` subpackages are included in installed artifacts. CI now builds and installs the wheel outside the source checkout before importing the core service modules. Added an architecture baseline, initial decision records, and executable dependency-direction guardrails for the existing modular-monolith boundaries. diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index 83cb9726..e1c439ac 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -27,6 +27,15 @@ from dataretrieval.ogc import OgcDialect, get_ogc_data, prepare_request_args from dataretrieval.utils import BaseMetadata +__all__ = [ + "get_sites", + "get_water_level", + "get_lithology", + "get_well_construction", + "get_providers", +] + + # The Water Data API base URL, defined locally to avoid importing policy internals. BASE_URL = "https://api.waterdata.usgs.gov" diff --git a/dataretrieval/nldi.py b/dataretrieval/nldi.py index 9a169414..04a0b477 100644 --- a/dataretrieval/nldi.py +++ b/dataretrieval/nldi.py @@ -3,7 +3,16 @@ from json import JSONDecodeError from typing import Any, Literal, cast -from dataretrieval.utils import query +from dataretrieval.utils import _query_with_retry + +__all__ = [ + "get_flowlines", + "get_basin", + "get_features", + "get_features_by_data_source", + "search", +] + try: import geopandas as gpd @@ -23,7 +32,7 @@ def _query_nldi( # A helper function to query the NLDI API. ``query()`` already raises a # typed ``DataRetrievalError`` for any HTTP error response, so a returned # response is a success that we only need to parse. - response = query(url, payload=query_params) + response = _query_with_retry(url, payload=query_params) response_data: dict[str, Any] | list[Any] = {} try: response_data = response.json() diff --git a/dataretrieval/ogc/__init__.py b/dataretrieval/ogc/__init__.py index 46dd3918..ab3502cd 100644 --- a/dataretrieval/ogc/__init__.py +++ b/dataretrieval/ogc/__init__.py @@ -8,10 +8,9 @@ - :func:`fetch_ogc_request` — execute a pre-built request with pagination. Service adapters (NGWMN, Water Data's generic wrapper) import from this -facade rather than reaching into engine internals. The engine module remains -available for lower-level orchestration needs (e.g. ``_paginate``, -``_run_sync``) that sibling modules like ``wateruse`` use under the accepted -temporary variance. +facade rather than reaching into engine internals. Generic execution policy +lives in :mod:`dataretrieval.transport`; the engine retains compatibility +wrappers at previous private paths. """ from dataretrieval.ogc.engine import fetch_ogc_request, get_ogc_data diff --git a/dataretrieval/ogc/chunking.py b/dataretrieval/ogc/chunking.py index 79037a6b..143ca94a 100644 --- a/dataretrieval/ogc/chunking.py +++ b/dataretrieval/ogc/chunking.py @@ -17,15 +17,13 @@ This module owns the *execution* half — the event loop and bounded concurrency that drive a plan to completion (``ChunkedCall``) plus the -public ``multi_value_chunked`` decorator. The neighboring concerns live in -sibling modules it imports, each with its own reason to change: -:mod:`~dataretrieval.ogc.planning` builds the -:class:`~dataretrieval.ogc.planning.ChunkPlan` and recombines per-chunk -frames and responses (pure, no I/O); :mod:`~dataretrieval.ogc.retry` holds -the transient-classification and exponential-backoff policy; and +public ``multi_value_chunked`` decorator. The neighboring concerns remain +separate: :mod:`~dataretrieval.ogc.planning` builds the +:class:`~dataretrieval.ogc.planning.ChunkPlan`; +:mod:`~dataretrieval.transport.combining` assembles results; +:mod:`~dataretrieval.transport.retry` owns bounded retry policy; and :mod:`~dataretrieval.ogc.interruptions` defines the resumable -:class:`~dataretrieval.ogc.interruptions.ChunkInterrupted` exception -contract. +:class:`~dataretrieval.ogc.interruptions.ChunkInterrupted` contract. Concurrency: ``multi_value_chunked`` fans every pending sub-request out under one ``asyncio.gather`` sharing a single ``httpx.AsyncClient``. An @@ -83,23 +81,19 @@ import pandas as pd from anyio.from_thread import start_blocking_portal -from dataretrieval.utils import HTTPX_ASYNC_DEFAULTS, Ambient, _require_positive_int - -from . import progress as _progress -from .combining import ( +from dataretrieval.transport import progress as _progress +from dataretrieval.transport.combining import ( _combine_chunk_frames, _combine_chunk_responses, ) -from .interruptions import ( - ChunkInterrupted, -) +from dataretrieval.transport.http import open_async_client +from dataretrieval.transport.retry import _NO_RETRY, RetryPolicy +from dataretrieval.transport.retry import retry_async as _retry +from dataretrieval.utils import Ambient, _require_positive_int + +from .interruptions import ChunkInterrupted from .planning import ChunkPlan -from .retry import ( - _NO_RETRY, - RetryPolicy, - _classify_chunk_error, - _retry, -) +from .retry import _classify_chunk_error # Empirically the API replies HTTP 414 above ~8200 bytes of full URL — # matches nginx's default ``large_client_header_buffers`` of 8 KB. 8000 @@ -650,7 +644,7 @@ async def _run(self, max_concurrent: int | None) -> tuple[pd.DataFrame, Any]: self.plan.total if max_concurrent is None else max_concurrent ) - async with httpx.AsyncClient(limits=limits, **HTTPX_ASYNC_DEFAULTS) as client: + async with open_async_client(limits=limits) as client: with _chunked_client(client): reporter = _progress.current() if reporter is not None: diff --git a/dataretrieval/ogc/combining.py b/dataretrieval/ogc/combining.py index be4366c1..e01ac4ab 100644 --- a/dataretrieval/ogc/combining.py +++ b/dataretrieval/ogc/combining.py @@ -1,206 +1,21 @@ -"""Result recombination: merge per-chunk frames and responses (no I/O). - -These utilities assemble the output of a chunked/fan-out call from its -individual per-sub-request results. They have no event-loop, retry, or -network state — they're pure data transforms imported by both the -chunked-call execution (:mod:`dataretrieval.ogc.chunking`) and the -per-page pagination (:mod:`dataretrieval.ogc.engine`). - -Separated from :mod:`dataretrieval.ogc.planning` so that module stays -focused on *what* to split, while this module owns *how* to reassemble. -""" - -from __future__ import annotations - -import copy -from datetime import timedelta - -import httpx -import pandas as pd - -# Response header USGS uses to advertise remaining hourly quota. Lives in this -# module so every layer (the combine helpers below, the engine's per-page -# progress reporter) reads it from one place rather than hard-coding the string. -_QUOTA_HEADER = "x-ratelimit-remaining" - - -def _safe_elapsed(response: httpx.Response) -> timedelta: - """ - Read ``response.elapsed``, falling back to ``timedelta(0)`` when - the attribute hasn't been populated. - - httpx only writes ``.elapsed`` when a response is closed through - its normal transport path. ``MockTransport`` (used by - ``pytest-httpx``) and hand-constructed ``httpx.Response`` objects - leave the attribute unset, so accessing it raises ``RuntimeError``. - Combining responses across chunks needs a defined duration, so we - treat the missing attribute as zero elapsed. - """ - try: - elapsed: object = response.elapsed - except RuntimeError: - return timedelta(0) - return elapsed if isinstance(elapsed, timedelta) else timedelta(0) - - -def _set_response_url(response: httpx.Response, url: str | httpx.URL) -> None: - """ - Overwrite the URL surfaced by a response without back-propagating - the change into any aliased original. - - Lightweight test doubles expose ``.url`` as a writable attribute. Real - :class:`httpx.Response` objects resolve it through a bound request, so swap - in a fresh request carrying the new URL; mutating the existing request would - leak through any shallow copy that shares it. - """ - if not isinstance(response, httpx.Response): - # Lightweight test doubles expose ``url`` as a writable attribute. - response.url = url - return - - target = httpx.URL(str(url)) - try: - old = response.request - except RuntimeError: - # No request bound (some hand-built httpx.Response fixtures); - # synthesize a minimal one to hold the URL. - response.request = httpx.Request("GET", target) - return - response.request = httpx.Request(method=old.method, url=target, headers=old.headers) - - -def _lowest_remaining(responses: list[httpx.Response]) -> httpx.Response: - """The response reporting the lowest ``x-ratelimit-remaining``. - - Within a rate-limit window, the counter decreases monotonically, so the - smallest value observed is the most conservative value to surface. Under - concurrent fan-out, the last response *by index* need not be the one the - server processed last. Fall back to the last response when none reports - the header. - """ - best: httpx.Response | None = None - best_remaining: int | None = None - for response in responses: - try: - remaining = int(response.headers[_QUOTA_HEADER]) - except (KeyError, ValueError): - continue - if best_remaining is None or remaining < best_remaining: - best, best_remaining = response, remaining - return best if best is not None else responses[-1] - - -def _merge_response( - base: httpx.Response, - *, - headers_from: httpx.Response, - elapsed: timedelta, - url: str | httpx.URL | None = None, -) -> httpx.Response: - """Fold several responses into one: a shallow copy of ``base`` whose - ``.headers`` are rebuilt as a fresh ``httpx.Headers`` from ``headers_from``, - ``.elapsed`` set to ``elapsed``, and ``.url`` overridden when ``url`` is - given. ``base`` and ``headers_from`` are never mutated, and the fresh - ``httpx.Headers`` means downstream mutations don't back-propagate into any - underlying response — so callers may re-fold idempotently. This is the one - low-level merge behind both pagination - (:func:`~dataretrieval.ogc.engine._paginate`) and the chunked / fan-out - aggregation (:func:`_combine_chunk_responses`).""" - merged = copy.copy(base) - merged.headers = httpx.Headers(headers_from.headers) - merged.elapsed = elapsed - if url is not None: - _set_response_url(merged, url) - return merged - - -def _combine_chunk_frames(frames: list[pd.DataFrame]) -> pd.DataFrame: - """Concatenate per-chunk frames and deduplicate IDs across chunks. - - Empty frames are ignored before concatenation so an empty plain - :class:`pandas.DataFrame` cannot downgrade a real ``GeoDataFrame`` and - strip its geometry or CRS. When every frame is empty, the first frame is - returned to preserve its concrete type. - - When multiple non-empty frames are combined, non-null feature IDs are - deduplicated regardless of the plan axis. Filter clauses can match the same - feature, and list inputs can contain repeated values or otherwise select - overlapping records. Rows without an ``id`` are preserved verbatim: pandas - treats null values as duplicates, so deduplicating them would silently lose - data. - """ - non_empty = [frame for frame in frames if not frame.empty] - if not non_empty: - return frames[0] if frames else pd.DataFrame() - if len(non_empty) == 1: - return non_empty[0].copy() - - combined = pd.concat(non_empty, ignore_index=True) - if "id" not in combined.columns: - return combined - - has_id = combined["id"].notna() - if has_id.all(): - return combined.drop_duplicates(subset="id", ignore_index=True) - if has_id.any(): - id_rows = combined[has_id].drop_duplicates(subset="id") - no_id_rows = combined[~has_id] - return pd.concat([id_rows, no_id_rows], ignore_index=True) - return combined - - -def _combine_chunk_responses( - responses: list[httpx.Response], canonical_url: str | None -) -> httpx.Response: - """ - Fold per-sub-request responses into a single aggregated response. - - For a multi-response input, returns a shallow copy of - ``responses[0]`` with ``.headers`` set to those of the response reporting - the lowest ``x-ratelimit-remaining`` value (the most conservative quota - observation; see :func:`_lowest_remaining`), ``.elapsed`` set to the sum of - the per-response elapsed durations, and ``.url`` set to the - canonical original-query URL (when supplied) so ``BaseMetadata`` - reflects the user's full request rather than the first chunk. - - For a single-response input with no canonical-URL override, - ``responses[0]`` is returned unchanged to skip the copy on the - passthrough hot path. - - Parameters - ---------- - responses : list[httpx.Response] - One response per completed sub-request, in caller-provided order. - canonical_url : str or None - URL of the unchunked original request. ``None`` skips the URL - override — used by the passthrough path (the fetcher's - response already carries the original-query URL) and by the - worst-case overflow path (no buildable canonical URL exists). - - Returns - ------- - httpx.Response - A shallow copy of the first response with aggregated - ``headers``, ``elapsed``, and ``url``. The function is - idempotent (the input responses' ``headers`` / ``elapsed`` / - ``url`` are never mutated), so it's safe to call repeatedly - via :attr:`ChunkedCall.partial_response` during error - inspection or resume retries. ``headers`` on the returned - object is a fresh ``httpx.Headers``, so mutations there don't - back-propagate into any chunk's underlying response. - """ - if len(responses) == 1 and canonical_url is None: - return responses[0] - - # Headers come from the response with the lowest reported remaining quota; - # ``_lowest_remaining`` returns the lone response as-is - # for a single-element list). ``_merge_response`` re-sums elapsed onto a - # fresh copy, so repeated calls (e.g. via ``ChunkedCall.partial_response`` - # during resume) stay idempotent. - elapsed = sum((_safe_elapsed(r) for r in responses), start=timedelta()) - return _merge_response( - responses[0], - headers_from=_lowest_remaining(responses), - elapsed=elapsed, - url=canonical_url, - ) +"""Compatibility imports for response aggregation now owned by transport.""" + +from dataretrieval.transport.combining import ( + _QUOTA_HEADER, + _combine_chunk_frames, + _combine_chunk_responses, + _lowest_remaining, + _merge_response, + _safe_elapsed, + _set_response_url, +) + +__all__ = [ + "_QUOTA_HEADER", + "_combine_chunk_frames", + "_combine_chunk_responses", + "_lowest_remaining", + "_merge_response", + "_safe_elapsed", + "_set_response_url", +] diff --git a/dataretrieval/ogc/context.py b/dataretrieval/ogc/context.py new file mode 100644 index 00000000..8b607509 --- /dev/null +++ b/dataretrieval/ogc/context.py @@ -0,0 +1,15 @@ +"""Ambient per-call OGC request context.""" + +from dataretrieval.ogc.policy import DEFAULT_DIALECT, OGC_API_URL, OgcDialect +from dataretrieval.utils import Ambient + +# Optional cap on rows accumulated by one paginated request. +_row_cap: Ambient[int | None] = Ambient("ogc_row_cap", None) + +# OGC base URL targeted by request construction and schema lookup. +_ogc_base_url: Ambient[str] = Ambient("ogc_base_url", OGC_API_URL) + +# Per-call request and response dialect. +_dialect: Ambient[OgcDialect] = Ambient("ogc_dialect", DEFAULT_DIALECT) + +__all__: list[str] = [] diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index 31b81f5f..f95f8950 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -1,8 +1,8 @@ """Generic OGC API engine shared by the Water Data and NGWMN getters. -This module holds the API-agnostic orchestration core for talking to an OGC -API Features service — async pagination, the sync bridge, and the chunked -fetch entry point :func:`get_ogc_data` that orchestrates them. Request +This module holds OGC API Features orchestration — OGC cursor/response +strategies and the chunked fetch entry point :func:`get_ogc_data`. Generic +pagination and sync dispatch live in :mod:`dataretrieval.transport`; request construction lives in :mod:`~dataretrieval.ogc.requests`. The surrounding concerns live in sibling modules it composes, each with its own reason to change: :mod:`~dataretrieval.ogc.dates` (time-parameter marshalling), @@ -27,23 +27,18 @@ import functools import logging from collections.abc import ( - AsyncIterator, Awaitable, Callable, ) -from contextlib import asynccontextmanager from typing import Any, TypeVar, cast import httpx import pandas as pd -from anyio.from_thread import start_blocking_portal import dataretrieval.ogc.chunking as chunking -import dataretrieval.ogc.progress as _progress -from dataretrieval.exceptions import DataRetrievalError +import dataretrieval.transport.progress as _progress from dataretrieval.ogc.chunking import get_active_client -from dataretrieval.ogc.combining import _QUOTA_HEADER, _merge_response, _safe_elapsed -from dataretrieval.ogc.errors import _paginated_failure_message, _raise_for_non_200 +from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.policy import ( BASE_URL, # noqa: F401 — compatibility alias DEFAULT_DIALECT, @@ -71,11 +66,11 @@ prepare_request_args, ) from dataretrieval.ogc.shaping import GEOPANDAS, _finalize_ogc, _get_resp_data +from dataretrieval.transport.pagination import paginate +from dataretrieval.transport.sync import run_sync from dataretrieval.utils import ( - HTTPX_ASYNC_DEFAULTS, BaseMetadata, _default_headers, # noqa: F401 — compatibility re-export for tests - _network_error, _require_positive_int, ) @@ -159,46 +154,6 @@ def _next_req_url( return None -@asynccontextmanager -async def _client_for( - client: httpx.AsyncClient | None, -) -> AsyncIterator[httpx.AsyncClient]: - """ - Yield a usable async client, picking the best available source. - - Resolution order: - - 1. ``client`` if the caller supplied one (borrowed; not closed - here — the caller owns its lifecycle). - 2. The chunker's shared async client if we're inside a - :class:`~dataretrieval.ogc.chunking.ChunkedCall` run (per - :func:`chunking.get_active_client`). Borrowed; the chunker - closes it on exit. - 3. A fresh short-lived ``httpx.AsyncClient`` opened here and closed - on context exit. - - Parameters - ---------- - client : httpx.AsyncClient or None - A caller-owned client to borrow, or ``None`` to defer to the - chunker's shared client or a temporary one. - - Yields - ------ - httpx.AsyncClient - The chosen client. - """ - if client is not None: - yield client - return - shared = get_active_client() - if shared is not None: - yield shared - return - async with httpx.AsyncClient(**HTTPX_ASYNC_DEFAULTS) as new: - yield new - - _Cursor = TypeVar("_Cursor") @@ -210,136 +165,16 @@ async def _paginate( client: httpx.AsyncClient | None = None, raise_for_status: Callable[[httpx.Response], None] = _raise_for_non_200, ) -> tuple[pd.DataFrame, httpx.Response]: - """ - Drive a paginated request to completion over an - :class:`httpx.AsyncClient`. - - The common shape behind the paginated fetch paths (e.g. - :func:`_walk_pages`): send the initial request, then loop calling - ``follow_up`` until ``parse_response`` reports a ``None`` cursor, - accumulating frames and elapsed time. Any mid-pagination failure - raises ``DataRetrievalError`` wrapping the cause — the API exposes no - resume cursor, so the caller's only recovery is to retry the whole - call. Issuing HTTP asynchronously lets the multiple sub-requests of a - chunked call run concurrently under - :meth:`~dataretrieval.ogc.chunking.ChunkedCall._run`. - - Parameters - ---------- - initial_req : httpx.Request - First-page request to send. - parse_response : callable - ``resp -> (df, next_cursor_or_None)``. Returns the page's - DataFrame and the cursor (URL, token, …) used to drive - ``follow_up`` for the next page; ``None`` terminates the loop. - follow_up : callable - ``(cursor, client) -> Awaitable[httpx.Response]``. Builds and - sends the next-page request. - client : httpx.AsyncClient, optional - Caller-borrowed client. ``None`` (default) means use the - chunker's shared client (if inside a chunked call) or open - a temporary one. - raise_for_status : callable, optional - ``resp -> None``; raises the typed error for a non-OK response. - Defaults to :func:`_raise_for_non_200` (the OGC ``{code, description}`` - envelope); wateruse passes its own to surface the NWDC ``detail``. - - Returns - ------- - df : pandas.DataFrame - Concatenation of every page's parsed frame. - response : httpx.Response - A shallow copy of the first-page response, with ``.headers`` - rebuilt as a fresh ``httpx.Headers`` reflecting the last page and - ``.elapsed`` set to the sum of the per-page response durations. The - canonical URL is preserved from the first page. The original first-page - response is not mutated. - - Raises - ------ - DataRetrievalError - On a non-200 initial response, the typed subclass for the status from - :func:`_raise_for_non_200` (a - :class:`~dataretrieval.exceptions.TransientError` for a retryable - 429 / 5xx, otherwise a fatal :class:`~dataretrieval.exceptions.HTTPError`); - or, on an initial-page parse failure or any subsequent-page failure, a - base ``DataRetrievalError`` wrapping the cause (built by - :func:`_paginated_failure_message`, original exception on ``__cause__``). - httpx.HTTPError - Network-level failures on the *initial* request (e.g. - ``ConnectError``, ``TimeoutException``) propagate unmodified - so callers can branch on the specific type; equivalent - failures on subsequent pages are wrapped per above. - """ - logger.debug("Requesting: %s", initial_req.url) - reporter = _progress.current() - - def report_page(page: httpx.Response, frame: pd.DataFrame) -> None: - """Tick the ambient progress reporter (a no-op when unset) for one page.""" - if reporter is not None: - reporter.set_rate_remaining( - page.headers.get(_QUOTA_HEADER), - limit=page.headers.get("x-ratelimit-limit"), - ) - reporter.add_page(rows=len(frame)) - - async with _client_for(client) as sess: - resp = await sess.send(initial_req) - raise_for_status(resp) - initial_response = resp - total_elapsed = _safe_elapsed(resp) - - try: - df, cursor = parse_response(resp) - except Exception as e: # noqa: BLE001 - # Initial-page parse failures (malformed JSON, missing - # ``features``, schema drift) get the same wrapped-message - # treatment as follow-up failures so callers see a consistent - # diagnostic regardless of which page broke. - logger.warning("Initial response parse failed.") - raise DataRetrievalError(_paginated_failure_message(0, e)) from e - dfs = [df] - # Stop following ``next`` links once the optional row cap is reached - # (see :func:`_row_cap`); ``None`` means uncapped. The concatenation - # is sliced to the cap below so a final over-budget page can't exceed it. - cap = _row_cap.get() - nrows = len(df) - # Guard a non-advancing or cyclic cursor (a server bug that would - # otherwise loop forever). OGC's next-URLs are unique, so this never - # fires for them; the Link-header pagers (e.g. wateruse) rely on it. - seen: set[Any] = set() - report_page(resp, df) - while ( - cursor is not None and cursor not in seen and (cap is None or nrows < cap) - ): - seen.add(cursor) - try: - resp = await follow_up(cursor, sess) - raise_for_status(resp) - df, cursor = parse_response(resp) - dfs.append(df) - nrows += len(df) - total_elapsed += _safe_elapsed(resp) - report_page(resp, df) - except Exception as e: # noqa: BLE001 - logger.warning( - "Request failed at cursor %r. Data download interrupted.", - cursor, - ) - raise DataRetrievalError(_paginated_failure_message(len(dfs), e)) from e - - # Fold the pages onto a COPY of the initial response so a caller that - # inspected it mid-pagination (a hook, a test fixture) never sees an - # in-place mutation. ``resp`` is the last page, whose headers carry the - # current ``x-ratelimit-remaining`` (monotonic, so the last page is the - # most depleted) — the same low-level merge the fan-out aggregation uses. - final_response = _merge_response( - initial_response, headers_from=resp, elapsed=total_elapsed - ) - result = pd.concat(dfs, ignore_index=True) - if cap is not None: - result = result.head(cap) - return result, final_response + """Compatibility wrapper around API-neutral cursor pagination.""" + active_client = client if client is not None else get_active_client() + return await paginate( + initial_req, + parse_response=parse_response, + follow_up=follow_up, + client=active_client, + raise_for_status=raise_for_status, + row_cap=_row_cap.get(), + ) def _ogc_parse_response( @@ -541,44 +376,12 @@ def _run_sync( service: str, error_url: str | httpx.URL | None = None, ) -> tuple[pd.DataFrame, httpx.Response]: - """Drive an async OGC fetch to completion from synchronous code. - - Opens the service progress context and runs ``make_coro()`` through a - short-lived ``anyio`` blocking portal (a worker thread), so the - non-chunked getters work whether or not the caller is already inside an - event loop (Jupyter/async apps). The portal copies the calling context, - so the active progress reporter still reaches the sub-requests. - - Shared by the non-chunked fetch paths; the chunked OGC getters - drive their own portal - inside :meth:`chunking.ChunkedCall.resume`. - - A connection failure on the initial request is surfaced as a typed - ``NetworkError`` against ``error_url`` when given (callers that build their - own requests, e.g. ``wateruse``), else the request-builder base the caller - scoped via ``_ogc_base_url`` (the OGC / NGWMN getters). - """ - with _progress.progress_context(service=service): - with start_blocking_portal() as portal: - try: - # ``portal.call`` is ``Any`` (anyio is skipped by mypy — its - # source uses 3.10 syntax our 3.9 target can't parse), so cast - # to the declared return type, as ``ChunkedCall`` does too. - return cast( - "tuple[pd.DataFrame, httpx.Response]", portal.call(make_coro) - ) - except httpx.TransportError as exc: - # The initial-request connection failure ``_paginate`` lets - # through raw; mid-pagination failures are already typed. - # Report the base URL actually targeted: callers that build - # their own requests (``wateruse``) pass ``error_url``; the OGC - # getters leave it unset and fall back to the request-builder - # base they scoped via ``_ogc_base_url`` (NGWMN/sibling APIs set - # their own), not a hardcoded host. - raise _network_error( - error_url if error_url is not None else _ogc_base_url.get(), - exc, - ) from exc + """Compatibility wrapper around the API-neutral sync bridge.""" + return run_sync( + make_coro, + service=service, + error_url=error_url if error_url is not None else _ogc_base_url.get(), + ) def fetch_ogc_request( diff --git a/dataretrieval/ogc/errors.py b/dataretrieval/ogc/errors.py index 0bb39b0e..90b541de 100644 --- a/dataretrieval/ogc/errors.py +++ b/dataretrieval/ogc/errors.py @@ -10,7 +10,11 @@ import httpx -from dataretrieval.exceptions import RateLimited, error_for_status +from dataretrieval.exceptions import error_for_status +from dataretrieval.transport.pagination import ( + paginated_failure_message as _paginated_failure_message, # noqa: F401 +) +from dataretrieval.transport.retry import parse_retry_after as _parse_retry_after def _error_body(resp: httpx.Response) -> str: @@ -64,37 +68,6 @@ def _error_body(resp: httpx.Response) -> str: ) -def _parse_retry_after(value: str | None) -> float | None: - """ - Parse a USGS ``Retry-After`` header into seconds. - - Parameters - ---------- - value : str or None - The raw header value, or ``None`` if absent. - - Returns - ------- - float or None - Non-negative delta-seconds, clamped at zero. ``None`` when the - header is absent or unparseable; ``ChunkedCall`` treats - ``None`` as "fall back to my own retry policy". - - Notes - ----- - USGS sends ``Retry-After`` as integer delta-seconds (empirically - verified — e.g. ``Retry-After: 2619``). The HTTP spec also allows - HTTP-date form, but USGS doesn't use it, so this function doesn't - bother parsing it. - """ - if not value: - return None - try: - return max(0.0, float(value.strip())) - except ValueError: - return None - - def _raise_for_non_200(resp: httpx.Response) -> None: """ Raise a typed exception for any non-200 response. @@ -129,43 +102,3 @@ def _raise_for_non_200(resp: httpx.Response) -> None: _error_body(resp), retry_after=_parse_retry_after(resp.headers.get("Retry-After")), ) - - -def _paginated_failure_message(pages_collected: int, cause: BaseException) -> str: - """ - Build a user-facing message for a mid-pagination failure. - - The API exposes no resume cursor, so the caller's only recovery is - to retry the whole call — the message lists the practical knobs, - tailored to whether the failure was rate-limit (429) or something - else. - - Parameters - ---------- - pages_collected : int - Number of pages successfully fetched before the failure. - cause : BaseException - The underlying exception that interrupted pagination. - - Returns - ------- - str - A message suitable for the ``DataRetrievalError`` that the - paginated fetch paths raise from the original exception. - """ - cause_str = str(cause).removesuffix(".") - # Some ``httpx`` exceptions (e.g. ``TimeoutException()`` with no args) - # stringify to empty; fall back to the class name so the - # returned message is always informative. - if not cause_str.strip(): - cause_str = type(cause).__name__ - if isinstance(cause, RateLimited): - action = "wait for the rate-limit window to reset and retry" - else: - action = "retry the request (possibly after a short backoff)" - return ( - f"Paginated request failed after collecting {pages_collected} " - f"page(s): {cause_str}. To recover: {action}, reduce the " - f"request size (e.g. fewer locations, a shorter time range, or " - f"a smaller ``limit``), or obtain an API token." - ) diff --git a/dataretrieval/ogc/planning.py b/dataretrieval/ogc/planning.py index 15b397a0..a4a09eee 100644 --- a/dataretrieval/ogc/planning.py +++ b/dataretrieval/ogc/planning.py @@ -4,16 +4,15 @@ deciding how to split one over-budget OGC request into URL-fitting sub-requests (:class:`ChunkPlan` and the axis/byte-accounting helpers). It has no event loop, retry policy, or network state — those live in -:mod:`dataretrieval.ogc.chunking` (execution) and -:mod:`dataretrieval.ogc.retry` (retry policy), which import the plan and +:mod:`dataretrieval.ogc.chunking` (resumable execution) and +:mod:`dataretrieval.transport.retry` (retry policy), which import the plan and drive it. Result recombination — reassembling the per-chunk frames and responses back into one result -(:func:`~dataretrieval.ogc.combining._combine_chunk_frames`, -:func:`~dataretrieval.ogc.combining._combine_chunk_responses`, etc.) — lives in -the sibling :mod:`dataretrieval.ogc.combining` module, which callers import -directly. +(:func:`~dataretrieval.transport.combining._combine_chunk_frames`, +:func:`~dataretrieval.transport.combining._combine_chunk_responses`, etc.) — +lives in the API-neutral :mod:`dataretrieval.transport.combining` module. """ from __future__ import annotations diff --git a/dataretrieval/ogc/progress.py b/dataretrieval/ogc/progress.py index 6177c30f..8080fa7f 100644 --- a/dataretrieval/ogc/progress.py +++ b/dataretrieval/ogc/progress.py @@ -1,293 +1,15 @@ -"""A single self-updating status line for paginated / chunked OGC queries. +"""Compatibility imports for progress reporting now owned by transport.""" -OGC getters fan out two ways the caller can't see: large multi-value -requests are split into URL-length-safe *chunks* (``chunking`` module), and each -request follows ``next`` links across an unknown number of *pages* -(``engine._paginate``). This module surfaces that work as one line on stderr, -rewritten in place as data arrives:: - - Retrieving: daily · 6 pages · 2,881 rows · 995/1,000 requests remaining - -The active reporter lives in a :class:`~contextvars.ContextVar` rather than being -threaded through every signature: progress is a cross-cutting concern that the -chunk orchestrator (outer, chunk counts) and the page-walking loop (inner, -page/row/rate-limit counts) both update without knowing about each other. Call -:func:`progress_context` to activate one and :func:`current` to reach it. - -By default the line is shown for interactive use — an interactive terminal or a -Jupyter/IPython kernel, like ``tqdm`` — while redirected logs and CI stay clean. -``API_USGS_PROGRESS`` forces it on (``1``/``true``) or off (``0``/``false``). -""" - -from __future__ import annotations - -import contextvars -import os -import sys -from collections.abc import Iterator -from contextlib import contextmanager -from typing import TextIO - - -def _group_int(value: str) -> str: - """Comma-group a plain ASCII integer string; pass anything else through. - - (``str.isdigit`` alone is True for non-decimal unicode digits that ``int`` - rejects, hence the ``isascii`` guard.) - """ - return f"{int(value):,}" if value.isascii() and value.isdigit() else value - - -# The reporter active for the current query. A ContextVar (not a module global) -# so the chunk orchestrator and the page loop resolve to the same reporter -# within one query, and an unrelated query in another context can't clobber its -# state. (It does not give concurrent queries sharing one stderr separate -# lines — they would still interleave.) -_active: contextvars.ContextVar[ProgressReporter | None] = contextvars.ContextVar( - "ogc_progress", default=None +from dataretrieval.transport.progress import ( + SIGNUP_URL, + ProgressReporter, + current, + progress_context, ) -# Where to register for an API key. Surfaced once when a query runs without an -# API key configured (no API_USGS_PAT), since unauthenticated callers hit much -# lower rate limits (see the API_USGS_PAT note in the README). -SIGNUP_URL = "https://api.waterdata.usgs.gov/signup/" - -# Process-level latch so the "no API key" pointer is shown at most once. -_api_key_hint_shown = False - - -def _in_jupyter_kernel() -> bool: - """True when running inside a Jupyter/IPython *kernel* (notebook, lab, - qtconsole). - - A kernel's ``stderr`` isn't a TTY, but it honors carriage-return rewrites in - the cell output area — the same mechanism ``tqdm`` rides on — so the line is - worth showing there. The plain IPython terminal REPL is a - ``TerminalInteractiveShell`` (already a TTY), so only the ZMQ kernel needs - this extra signal. Detected without importing IPython: if it isn't already - imported, we aren't in a shell. - """ - ipython = sys.modules.get("IPython") - if ipython is None: - return False - shell = ipython.get_ipython() - return shell is not None and type(shell).__name__ == "ZMQInteractiveShell" - - -def _enabled_default(stream: TextIO) -> bool: - """Whether to draw the line by default. - - ``API_USGS_PROGRESS`` wins when set. Otherwise show it for interactive use — - a TTY or a Jupyter/IPython kernel — and stay quiet for redirected output, - logs, and CI. - """ - override = os.getenv("API_USGS_PROGRESS") - if override is not None: - return override.strip().lower() not in {"", "0", "false", "no", "off"} - if _in_jupyter_kernel(): - return True - return hasattr(stream, "isatty") and stream.isatty() - - -class ProgressReporter: - """Accumulates query progress and rewrites a single status line in place. - - Every update method is a no-op when the reporter is disabled, so call sites - need no ``if enabled`` guards. The line is redrawn with a leading carriage - return and padded to erase the previous (possibly longer) contents; - :meth:`close` terminates it with a newline so the final state persists. - """ - - def __init__( - self, - *, - service: str | None = None, - stream: TextIO | None = None, - enabled: bool | None = None, - ) -> None: - self._stream = stream if stream is not None else sys.stderr - self.enabled = _enabled_default(self._stream) if enabled is None else enabled - # The service/collection being retrieved (e.g. "daily", "peaks"), - # shown as the line's leading label. - self.service = service - self.total_chunks = 1 - self.current_chunk = 0 - self.pages = 0 - self.rows = 0 - self.rate_remaining: str | None = None - # The hourly request quota (``x-ratelimit-limit``), shown as the - # denominator when the server reports it. - self.rate_limit: str | None = None - # Transient note shown while a sub-request backs off before a - # retry; cleared by the next page/chunk so it doesn't linger. - self.retry_note: str | None = None - self._last_len = 0 - # Whether anything was actually written to the stream — drives whether - # close() needs a terminating newline. (``current_chunk`` is a poor - # proxy: ``start_chunk`` sets it even when it doesn't render.) - self._rendered = False - self._closed = False - - def set_chunks(self, total: int) -> None: - """Record how many filter chunks this query was split into.""" - self.total_chunks = max(int(total), 1) - - def start_chunk(self, index: int) -> None: - """Mark the start of chunk ``index`` (1-based) and redraw. - - Only redraws when actually chunking (``total_chunks > 1``); a - single-chunk plan has nothing chunk-specific to show yet, so it - avoids a premature "0 pages" frame before the first page arrives. - """ - self.current_chunk = index - self.retry_note = None - if self.total_chunks > 1: - self._render() - - def add_page(self, rows: int = 0) -> None: - """Record one fetched page carrying ``rows`` rows and redraw.""" - self.pages += 1 - self.rows += int(rows) - self.retry_note = None - self._render() - - def note_retry(self, *, attempt: int, wait: float) -> None: - """Show that a sub-request is backing off before retry ``attempt``. - - Cleared by the next :meth:`add_page` / :meth:`start_chunk` (or by - :meth:`close`) so the line returns to normal once the retry resolves. - """ - # Keep sub-second waits explicit (avoid misleading ``0s``) while - # rendering whole-second waits without unnecessary ``.0`` noise. - # ``float()`` to support Python 3.9-3.11: ``round(int, 1)`` returns an - # int and ``int.is_integer()`` (used below) only exists on 3.12+. - wait_1dp = round(float(wait), 1) - if wait_1dp < 1 or not wait_1dp.is_integer(): - secs = f"{wait_1dp:.1f}s" - else: - secs = f"{wait_1dp:.0f}s" - self.retry_note = f"retrying (attempt {attempt}, waiting {secs})" - self._render() - - def set_rate_remaining( - self, value: str | int | None, limit: str | int | None = None - ) -> None: - """Update the rate-limit display from the response headers. - - ``value`` is ``x-ratelimit-remaining``; ``limit`` is the optional - ``x-ratelimit-limit`` quota, shown as the denominator. Empty/missing - values are ignored so a page that omits a header doesn't blank out the - last known value. - """ - if value not in (None, ""): - self.rate_remaining = str(value) - if limit not in (None, ""): - self.rate_limit = str(limit) - - def _format(self) -> str: - parts: list[str] = [] - if self.total_chunks > 1: - parts.append(f"chunk {self.current_chunk}/{self.total_chunks}") - parts.append(f"{self.pages} page" + ("" if self.pages == 1 else "s")) - if self.rows: - parts.append(f"{self.rows:,} rows") - if self.rate_remaining is not None: - remaining = _group_int(self.rate_remaining) - if self.rate_limit is not None: - limit = _group_int(self.rate_limit) - segment = f"{remaining}/{limit} requests remaining" - else: - segment = f"{remaining} requests remaining" - parts.append(segment) - if self.retry_note is not None: - parts.append(self.retry_note) - if self.service: - return f"Retrieving: {self.service} · " + " · ".join(parts) - return "Progress: " + " · ".join(parts) - - def _render(self) -> None: - if not self.enabled or self._closed: - return - try: - line = self._format() - pad = max(self._last_len - len(line), 0) - self._stream.write("\r" + line + " " * pad) - self._stream.flush() - self._last_len = len(line) - self._rendered = True - except Exception: # noqa: BLE001 - # Progress output is best-effort cosmetics; a broken pipe (output - # piped to ``head``), a closed stream, or an encoding error must - # never disturb — let alone truncate — the query. Disable so we - # don't retry on every subsequent page. - self.enabled = False - - def close(self) -> None: - """Finalize the line with a trailing newline so it persists on screen. - - If no API key is configured (no ``API_USGS_PAT``), append a one-time - pointer to API-key registration, since unauthenticated callers hit much - lower rate limits. - """ - if self._closed: - return - # A retry note set during the final backoff would otherwise freeze as - # the persisted last line of a call that has since completed or given - # up; clear it and redraw (while still un-closed, so ``_render`` runs) - # so the final state isn't a stale "retrying". - if self.enabled and self._rendered and self.retry_note is not None: - self.retry_note = None - self._render() - self._closed = True - if not (self.enabled and self._rendered): - return - try: - self._stream.write("\n") - self._maybe_hint_api_key() - self._stream.flush() - except Exception: # noqa: BLE001 - self.enabled = False - - def _maybe_hint_api_key(self) -> None: - global _api_key_hint_shown - if _api_key_hint_shown or os.getenv("API_USGS_PAT"): - return - # Set the once-per-process latch only after a successful write, so a - # failed write (broken pipe) doesn't silently burn the hint for every - # later query in the process. - self._stream.write( - f"No API key detected — register for higher rate limits at {SIGNUP_URL}\n" - ) - _api_key_hint_shown = True - - -@contextmanager -def progress_context( - *, - service: str | None = None, - stream: TextIO | None = None, - enabled: bool | None = None, -) -> Iterator[ProgressReporter]: - """Activate a :class:`ProgressReporter` for the duration of a query. - - ``service`` labels the line (e.g. ``"Retrieving: daily ..."``). If a reporter - is already active (a nested call), the existing one is yielded unchanged so - the outermost query owns the single line; only the outermost context closes - it (and ``service``/``stream``/``enabled`` of a nested call are ignored). - """ - existing = _active.get() - if existing is not None: - yield existing - return - reporter = ProgressReporter(service=service, stream=stream, enabled=enabled) - token = _active.set(reporter) - try: - yield reporter - finally: - _active.reset(token) - reporter.close() - - -def current() -> ProgressReporter | None: - """Return the reporter active for the current query, or ``None``.""" - return _active.get() +__all__ = [ + "SIGNUP_URL", + "ProgressReporter", + "current", + "progress_context", +] diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index eb5a201b..2efc7a02 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -1,49 +1,29 @@ -"""OGC request preparation, construction, and schema/queryables lookup. +"""OGC argument normalization and HTTP request construction. -This module owns the machinery for building OGC API requests (both GET and -POST/CQL2 paths), the ambient base-URL and dialect state that request builders -read, and the queryables/schema request helper used by empty-result shaping. - -It depends on :mod:`~dataretrieval.ogc.policy` (the dialect type and endpoint -constants), :mod:`~dataretrieval.ogc.dates`, :mod:`~dataretrieval.ogc.errors`, -and :mod:`~dataretrieval.utils` (shared HTTP primitives). It must NOT import -engine or shaping. +Ambient request state lives in :mod:`dataretrieval.ogc.context`; queryables and +schema execution live in :mod:`dataretrieval.ogc.schema`. The schema helper is +imported here only to preserve its previous private path. """ from __future__ import annotations import json -import logging import re from collections.abc import Iterable, Mapping -from typing import Any, cast +from typing import Any import httpx +from dataretrieval.ogc.context import _dialect as _dialect +from dataretrieval.ogc.context import _ogc_base_url as _ogc_base_url +from dataretrieval.ogc.context import _row_cap as _context_row_cap from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS, _format_api_dates -from dataretrieval.ogc.errors import _raise_for_non_200 -from dataretrieval.ogc.policy import DEFAULT_DIALECT, OGC_API_URL, OgcDialect -from dataretrieval.utils import HTTPX_DEFAULTS, Ambient, _default_headers, _get - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Ambient per-call state -# --------------------------------------------------------------------------- - -# Optional cap on the rows one paginated call accumulates before it stops -# following ``next`` links (``None`` = uncapped). Set by :func:`get_reference_table` -# to preview large tables without downloading every page. -_row_cap: Ambient[int | None] = Ambient("ogc_row_cap", None) - -# OGC base URL the shared request builder (:func:`_construct_api_requests`) -# targets — the main Water Data API or, for NGWMN collections, their own base. -_ogc_base_url: Ambient[str] = Ambient("ogc_base_url", OGC_API_URL) - -# Per-call OGC dialect the request builder reads for CQL2-vs-GET routing and -# date-only formatting (default: a plain OGC API). -_dialect: Ambient[OgcDialect] = Ambient("ogc_dialect", DEFAULT_DIALECT) +from dataretrieval.ogc.schema import _check_ogc_requests as _schema_check_ogc_requests +from dataretrieval.transport.http import default_headers as _default_headers +# Previous private paths remain available while ownership lives in context/schema. +_row_cap = _context_row_cap +_check_ogc_requests = _schema_check_ogc_requests # --------------------------------------------------------------------------- # Monitoring location ID validation @@ -212,18 +192,6 @@ def _construct_cql_request( ) -def _check_ogc_requests( - endpoint: str, req_type: str = "queryables" -) -> tuple[dict[str, Any], httpx.Response]: - """Send an HTTP GET request to the OGC endpoint for queryables/schema.""" - if req_type not in ("queryables", "schema"): - raise ValueError(f"req_type must be 'queryables' or 'schema', got {req_type!r}") - url = f"{_ogc_base_url.get()}/collections/{endpoint}/{req_type}" - resp = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) - _raise_for_non_200(resp) - return cast("dict[str, Any]", resp.json()), resp - - # --------------------------------------------------------------------------- # Argument normalization helpers # --------------------------------------------------------------------------- diff --git a/dataretrieval/ogc/retry.py b/dataretrieval/ogc/retry.py index bd45f275..f2b45a77 100644 --- a/dataretrieval/ogc/retry.py +++ b/dataretrieval/ogc/retry.py @@ -1,211 +1,39 @@ -"""Transient-failure retry policy for chunked sub-requests. - -Defines what counts as a retryable transient (:func:`_classify_chunk_error`, -:func:`_retryable`), the bounded exponential-backoff-with-jitter policy -(:class:`RetryPolicy`), and the driver that applies it (:func:`_retry`). Kept -separate from the execution engine in :mod:`dataretrieval.ogc.chunking` so the -retry/backoff behavior is one cohesive unit that changes independently of the -concurrency model. -""" +"""OGC interruption classification over API-neutral transport retry policy.""" from __future__ import annotations -import asyncio -import os -import random -from collections.abc import Awaitable, Callable -from dataclasses import dataclass - import httpx -import pandas as pd -import dataretrieval.ogc.progress as _progress from dataretrieval.exceptions import RateLimited, TransientError from dataretrieval.ogc.interruptions import ( ChunkInterrupted, QuotaExhausted, ServiceInterrupted, ) - -# Retry-with-backoff defaults for transient sub-request failures (429 / -# 5xx / connect-read timeouts): exponential backoff with full jitter, and -# honor a server ``Retry-After`` up to the cap below before escalating -# to a resumable interruption instead. -_RETRIES_ENV = "API_USGS_RETRIES" - - -_RETRIES_DEFAULT = 4 - - -_RETRY_BASE_BACKOFF = 0.5 - - -_RETRY_MAX_BACKOFF = 30.0 - - -_RETRY_AFTER_CAP = 60.0 - - -def _read_retries_env() -> int: - """ - Resolve the ``API_USGS_RETRIES`` env var to a max-retry count. - - Returns - ------- - int - Number of retries after the first attempt; ``0`` disables - retrying. Unset/blank → ``_RETRIES_DEFAULT``. - """ - raw = os.environ.get(_RETRIES_ENV) - if raw is None or raw.strip() == "": - return _RETRIES_DEFAULT - try: - value = int(raw.strip()) - except ValueError as exc: - raise ValueError( - f"{_RETRIES_ENV} must be a non-negative integer (got {raw!r})." - ) from exc - if value < 0: - raise ValueError(f"{_RETRIES_ENV} must be >= 0 (got {value}).") - return value - - -@dataclass(frozen=True) -class RetryPolicy: - """Bounded retry-with-backoff config for transient sub-request failures. - - An immutable value object that owns the *timing* decisions; the - exception taxonomy (which failures are retryable) lives in - :func:`_retryable`. Backoff is exponential with **full jitter** - (:func:`random.uniform` over ``[0, ceiling]``) so the concurrent - fan-out's retries don't re-burst in lockstep. A server ``Retry-After`` - hint, when present, overrides the computed backoff — unless it exceeds - :attr:`retry_after_cap`, in which case retrying stops and the failure - surfaces as a resumable :class:`ChunkInterrupted` (a multi-minute - quota-window reset shouldn't block the call inline). - - Attributes - ---------- - max_retries : int - Retries attempted after the first try; ``0`` disables retrying. - base_backoff : float - Seconds; the jitter ceiling for the first retry, doubled each - subsequent attempt. - max_backoff : float - Upper bound on any single attempt's backoff ceiling. - retry_after_cap : float - Largest ``Retry-After`` (seconds) honored inline; longer hints - escalate to a resumable interruption. - """ - - max_retries: int = _RETRIES_DEFAULT - base_backoff: float = _RETRY_BASE_BACKOFF - max_backoff: float = _RETRY_MAX_BACKOFF - retry_after_cap: float = _RETRY_AFTER_CAP - - def __post_init__(self) -> None: - # Catch invalid timing knobs here so a misconfiguration fails at - # construction, not deep in a later ``time.sleep`` (ValueError on - # a negative delay) or silently in ``asyncio.sleep`` (which - # treats negative as zero). - if self.max_retries < 0: - raise ValueError(f"max_retries must be >= 0 (got {self.max_retries}).") - if self.base_backoff < 0 or self.max_backoff < 0 or self.retry_after_cap < 0: - raise ValueError("retry backoff settings must be non-negative.") - - @classmethod - def from_env(cls) -> RetryPolicy: - """ - Build a policy from the module-level defaults, resolved now. - - Reads ``max_retries`` from ``API_USGS_RETRIES`` and the timing - knobs from the ``_RETRY_*`` module constants at call time — not - the dataclass field defaults (which freeze at class definition) - — so test ``monkeypatch.setattr`` on the constants takes effect. - - Returns - ------- - RetryPolicy - A policy built from the module-level defaults resolved at - call time. - """ - return cls( - max_retries=_read_retries_env(), - base_backoff=_RETRY_BASE_BACKOFF, - max_backoff=_RETRY_MAX_BACKOFF, - retry_after_cap=_RETRY_AFTER_CAP, - ) - - def should_retry(self, attempt: int, retry_after: float | None) -> bool: - """ - Whether a just-failed ``attempt`` (1-based) warrants another try. - - A ``Retry-After`` longer than ``retry_after_cap`` is *not* slept - off inline — it returns ``False`` so the failure escalates to a - resumable interruption instead of blocking the call for minutes. - - Parameters - ---------- - attempt : int - The just-failed attempt number (1-based). - retry_after : float or None - Seconds the server suggested waiting (``Retry-After`` hint), - or ``None`` when no hint was given. - - Returns - ------- - bool - ``True`` if another try is warranted, ``False`` otherwise. - """ - if attempt > self.max_retries: - return False - return retry_after is None or retry_after <= self.retry_after_cap - - def backoff(self, attempt: int, retry_after: float | None) -> float: - """ - Seconds to wait before retry ``attempt`` (1-based). - - Parameters - ---------- - attempt : int - The retry attempt number (1-based). - retry_after : float or None - Seconds the server suggested waiting (``Retry-After`` hint), - or ``None`` to use the computed exponential backoff instead. - - Returns - ------- - float - Seconds to wait before the retry. - """ - if retry_after is not None: - return retry_after - ceiling = min(self.max_backoff, self.base_backoff * 2 ** (attempt - 1)) - return random.uniform(0.0, ceiling) - - -# Default for direct ``ChunkedCall`` / ``ChunkPlan.execute`` construction -# (and tests): no retrying. The production decorator path explicitly passes -# ``RetryPolicy.from_env()`` so retries are on by default there. -_NO_RETRY = RetryPolicy(max_retries=0) +from dataretrieval.transport.retry import ( + _NO_RETRY, + _RETRIES_DEFAULT, + _RETRIES_ENV, + _RETRY_AFTER_CAP, + _RETRY_BASE_BACKOFF, + _RETRY_MAX_BACKOFF, + RetryPolicy, + _read_retries_env, + _retry_delay, + _retryable, +) +from dataretrieval.transport.retry import ( + retry_async as _retry, +) def _classify_transient( exc: BaseException, ) -> tuple[type[ChunkInterrupted], float | None] | None: - """Classify one exception as a transient, resumable failure. - - This function owns the shared exception taxonomy; it deliberately does not - walk ``__cause__``. :func:`_classify_chunk_error` walks wrapped pagination - failures, while :func:`_retryable` applies the narrower automatic-retry - policy to this classification. - """ + """Classify one failure as a resumable OGC interruption.""" if isinstance(exc, RateLimited): return QuotaExhausted, exc.retry_after if isinstance(exc, TransientError): - # Every typed transient other than a rate-limit error is a service - # interruption. This fallback keeps future TransientError subclasses - # resumable after their inline retries are exhausted. return ServiceInterrupted, exc.retry_after if isinstance(exc, (httpx.HTTPError, httpx.InvalidURL)): return ServiceInterrupted, None @@ -215,139 +43,28 @@ def _classify_transient( def _classify_chunk_error( exc: BaseException, ) -> tuple[type[ChunkInterrupted], float | None] | None: - """ - Classify a fetch error as a known transient (resumable) failure. - - Walks the ``__cause__`` chain of ``exc`` looking for a known typed - transport failure. Returns the matching ``ChunkInterrupted`` - subclass and any ``Retry-After`` hint, or ``None`` if the error is - not a recognized transient — in which case ``ChunkedCall`` - re-raises rather than wrapping (programmer errors and unknown - failures shouldn't masquerade as resumable). - - Parameters - ---------- - exc : BaseException - The exception raised by a sub-request. - - Returns - ------- - tuple[type[ChunkInterrupted], float or None] or None - ``(interrupted_class, retry_after)`` for recognized transient - failures; ``None`` otherwise. - - Notes - ----- - ``_walk_pages`` re-wraps mid-pagination failures as a base - ``DataRetrievalError`` with the typed transport exception linked as - ``__cause__``, so this function must walk the chain rather than - just ``isinstance`` the top-level exception. - - Bare ``httpx.HTTPError`` (``ConnectError``, ``TimeoutException``, - etc.) and ``httpx.InvalidURL`` (server-supplied cursor URL too - long, oversize follow-up) are also treated as transport failures - and wrapped as :class:`ServiceInterrupted` — they aren't one of the - typed status errors above (and ``InvalidURL`` doesn't even inherit - from ``httpx.HTTPError``), so without explicit handling they would - escape classification with no resumable handle. - """ - cur: BaseException | None = exc - while cur is not None: - result = _classify_transient(cur) + """Walk a wrapped pagination failure for a resumable transport cause.""" + current: BaseException | None = exc + while current is not None: + result = _classify_transient(current) if result is not None: return result - cur = cur.__cause__ + current = current.__cause__ return None -def _retryable(exc: BaseException) -> tuple[bool, float | None]: - """Decide whether a top-level transient is worth an automatic retry. - - Wrapped mid-pagination failures are not retried from page one; they instead - escalate to a resumable :class:`ChunkInterrupted`. ``httpx.InvalidURL`` and - non-transport ``httpx.HTTPError`` instances are classified as resumable but - excluded from automatic retry by policy. - """ - classification = _classify_transient(exc) - if classification is None: - return False, None - - _, retry_after = classification - if isinstance(exc, (TransientError, httpx.TransportError)): - return True, retry_after - return False, None - - -def _retry_delay(exc: BaseException, attempt: int, policy: RetryPolicy) -> float | None: - """ - Decide the backoff for a just-failed ``attempt`` (1-based), or ``None`` - to give up and re-raise. - - Returns ``None`` in three cases — the error isn't a retryable - transient, the policy is exhausted, or the server's ``Retry-After`` - exceeds the cap (escalates to a resumable :class:`ChunkInterrupted` - instead). Otherwise returns the seconds to wait and emits the - progress-bar retry note. - - Parameters - ---------- - exc : BaseException - The exception raised by the just-failed attempt. - attempt : int - The just-failed attempt number (1-based). - policy : RetryPolicy - The retry-with-backoff policy governing the decision. - - Returns - ------- - float or None - Seconds to wait before retrying, or ``None`` to give up and - re-raise. - """ - retryable, retry_after = _retryable(exc) - if not retryable or not policy.should_retry(attempt, retry_after): - return None - delay = policy.backoff(attempt, retry_after) - # Surface the imminent retry on the active progress reporter, if any. - reporter = _progress.current() - if reporter is not None: - reporter.note_retry(attempt=attempt, wait=delay) - return delay - - -async def _retry( - afn: Callable[[], Awaitable[tuple[pd.DataFrame, httpx.Response]]], - policy: RetryPolicy, -) -> tuple[pd.DataFrame, httpx.Response]: - """ - Call ``afn`` with bounded retry-with-backoff on transient failures. - - A non-retryable or policy-exhausted failure (see :func:`_retry_delay`) - propagates unchanged so the caller's existing handling wraps it as a - resumable :class:`ChunkInterrupted`. The whole retry *decision* lives - in :func:`_retry_delay`; this driver only awaits the sleep between - attempts. - - Parameters - ---------- - afn : Callable - Zero-arg awaitable callable that issues a single sub-request and - returns ``(frame, response)``. - policy : RetryPolicy - The retry-with-backoff policy governing the retries. - - Returns - ------- - tuple of (pandas.DataFrame, httpx.Response) - The ``(frame, response)`` pair from the first successful call. - """ - attempt = 0 - while True: - try: - return await afn() - except Exception as exc: # noqa: BLE001 — re-raised unless retryable - attempt += 1 - delay = _retry_delay(exc, attempt, policy) - if delay is None: - raise - await asyncio.sleep(delay) +__all__ = [ + "RetryPolicy", + "_NO_RETRY", + "_RETRIES_DEFAULT", + "_RETRIES_ENV", + "_RETRY_AFTER_CAP", + "_RETRY_BASE_BACKOFF", + "_RETRY_MAX_BACKOFF", + "_read_retries_env", + "_classify_chunk_error", + "_classify_transient", + "_retry", + "_retry_delay", + "_retryable", +] diff --git a/dataretrieval/ogc/schema.py b/dataretrieval/ogc/schema.py new file mode 100644 index 00000000..1b92ae64 --- /dev/null +++ b/dataretrieval/ogc/schema.py @@ -0,0 +1,28 @@ +"""OGC queryables and schema retrieval.""" + +from __future__ import annotations + +from typing import Any, cast + +import httpx + +from dataretrieval.ogc.context import _ogc_base_url +from dataretrieval.ogc.errors import _raise_for_non_200 +from dataretrieval.transport.http import HTTPX_DEFAULTS +from dataretrieval.transport.http import default_headers as _default_headers +from dataretrieval.transport.http import get as _get + + +def _check_ogc_requests( + endpoint: str, req_type: str = "queryables" +) -> tuple[dict[str, Any], httpx.Response]: + """Retrieve one collection's queryables or response schema.""" + if req_type not in ("queryables", "schema"): + raise ValueError(f"req_type must be 'queryables' or 'schema', got {req_type!r}") + url = f"{_ogc_base_url.get()}/collections/{endpoint}/{req_type}" + response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) + _raise_for_non_200(response) + return cast("dict[str, Any]", response.json()), response + + +__all__: list[str] = [] diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 383e54f1..00cd3470 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -184,7 +184,7 @@ def _deal_with_empty( if return_list.empty: if not properties or all(pd.isna(properties)): # Import from requests module (no engine dependency). - from dataretrieval.ogc.requests import _check_ogc_requests + from dataretrieval.ogc.schema import _check_ogc_requests schema, _ = _check_ogc_requests(endpoint=service, req_type="schema") properties = list(schema.get("properties", {}).keys()) diff --git a/dataretrieval/streamstats.py b/dataretrieval/streamstats.py index 1458494a..6681191f 100644 --- a/dataretrieval/streamstats.py +++ b/dataretrieval/streamstats.py @@ -12,7 +12,10 @@ import httpx -from dataretrieval.utils import HTTPX_DEFAULTS, _get, _raise_for_status +from dataretrieval.transport.http import HTTPX_DEFAULTS +from dataretrieval.utils import _get_with_retry + +__all__ = ["download_workspace", "get_sample_watershed", "get_watershed", "Watershed"] def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: @@ -37,9 +40,7 @@ def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: payload = {"workspaceID": workspaceID, "format": format} url = "https://streamstats.usgs.gov/streamstatsservices/download" - r = _get(url, params=payload, **HTTPX_DEFAULTS) - - _raise_for_status(r) + r = _get_with_retry(url, params=payload, **HTTPX_DEFAULTS) return r # data = r.raw.read() @@ -144,9 +145,7 @@ def get_watershed( } url = "https://streamstats.usgs.gov/streamstatsservices/watershed.geojson" - r = _get(url, params=payload, **HTTPX_DEFAULTS) - - _raise_for_status(r) + r = _get_with_retry(url, params=payload, **HTTPX_DEFAULTS) if format == "geojson": return r diff --git a/dataretrieval/transport/__init__.py b/dataretrieval/transport/__init__.py new file mode 100644 index 00000000..3739dcba --- /dev/null +++ b/dataretrieval/transport/__init__.py @@ -0,0 +1,9 @@ +"""Internal API-neutral HTTP transport and execution policy. + +The modules in this package own reusable client lifecycle, authentication, +pagination, retry, response aggregation, progress, and sync-dispatch behavior. +Service and protocol adapters consume these components; this package is not a +public framework API. +""" + +__all__: list[str] = [] diff --git a/dataretrieval/transport/combining.py b/dataretrieval/transport/combining.py new file mode 100644 index 00000000..0975da2f --- /dev/null +++ b/dataretrieval/transport/combining.py @@ -0,0 +1,206 @@ +"""Result recombination: merge per-chunk frames and responses (no I/O). + +These utilities assemble the output of a chunked/fan-out call from its +individual per-sub-request results. They have no event-loop, retry, or +network state — they're pure data transforms shared by protocol-specific +chunk execution, service fan-out, and +cursor-driven pagination. + +Separated from :mod:`dataretrieval.ogc.planning` so that module stays +focused on *what* to split, while this module owns *how* to reassemble. +""" + +from __future__ import annotations + +import copy +from datetime import timedelta + +import httpx +import pandas as pd + +# Response header USGS uses to advertise remaining hourly quota. Lives in this +# module so every layer (the combine helpers below, the engine's per-page +# progress reporter) reads it from one place rather than hard-coding the string. +_QUOTA_HEADER = "x-ratelimit-remaining" + + +def _safe_elapsed(response: httpx.Response) -> timedelta: + """ + Read ``response.elapsed``, falling back to ``timedelta(0)`` when + the attribute hasn't been populated. + + httpx only writes ``.elapsed`` when a response is closed through + its normal transport path. ``MockTransport`` (used by + ``pytest-httpx``) and hand-constructed ``httpx.Response`` objects + leave the attribute unset, so accessing it raises ``RuntimeError``. + Combining responses across chunks needs a defined duration, so we + treat the missing attribute as zero elapsed. + """ + try: + elapsed: object = response.elapsed + except RuntimeError: + return timedelta(0) + return elapsed if isinstance(elapsed, timedelta) else timedelta(0) + + +def _set_response_url(response: httpx.Response, url: str | httpx.URL) -> None: + """ + Overwrite the URL surfaced by a response without back-propagating + the change into any aliased original. + + Lightweight test doubles expose ``.url`` as a writable attribute. Real + :class:`httpx.Response` objects resolve it through a bound request, so swap + in a fresh request carrying the new URL; mutating the existing request would + leak through any shallow copy that shares it. + """ + if not isinstance(response, httpx.Response): + # Lightweight test doubles expose ``url`` as a writable attribute. + response.url = url + return + + target = httpx.URL(str(url)) + try: + old = response.request + except RuntimeError: + # No request bound (some hand-built httpx.Response fixtures); + # synthesize a minimal one to hold the URL. + response.request = httpx.Request("GET", target) + return + response.request = httpx.Request(method=old.method, url=target, headers=old.headers) + + +def _lowest_remaining(responses: list[httpx.Response]) -> httpx.Response: + """The response reporting the lowest ``x-ratelimit-remaining``. + + Within a rate-limit window, the counter decreases monotonically, so the + smallest value observed is the most conservative value to surface. Under + concurrent fan-out, the last response *by index* need not be the one the + server processed last. Fall back to the last response when none reports + the header. + """ + best: httpx.Response | None = None + best_remaining: int | None = None + for response in responses: + try: + remaining = int(response.headers[_QUOTA_HEADER]) + except (KeyError, ValueError): + continue + if best_remaining is None or remaining < best_remaining: + best, best_remaining = response, remaining + return best if best is not None else responses[-1] + + +def _merge_response( + base: httpx.Response, + *, + headers_from: httpx.Response, + elapsed: timedelta, + url: str | httpx.URL | None = None, +) -> httpx.Response: + """Fold several responses into one: a shallow copy of ``base`` whose + ``.headers`` are rebuilt as a fresh ``httpx.Headers`` from ``headers_from``, + ``.elapsed`` set to ``elapsed``, and ``.url`` overridden when ``url`` is + given. ``base`` and ``headers_from`` are never mutated, and the fresh + ``httpx.Headers`` means downstream mutations don't back-propagate into any + underlying response — so callers may re-fold idempotently. This is the one + low-level merge behind both pagination + (:func:`~dataretrieval.transport.pagination.paginate`) and the chunked / fan-out + aggregation (:func:`_combine_chunk_responses`).""" + merged = copy.copy(base) + merged.headers = httpx.Headers(headers_from.headers) + merged.elapsed = elapsed + if url is not None: + _set_response_url(merged, url) + return merged + + +def _combine_chunk_frames(frames: list[pd.DataFrame]) -> pd.DataFrame: + """Concatenate per-chunk frames and deduplicate IDs across chunks. + + Empty frames are ignored before concatenation so an empty plain + :class:`pandas.DataFrame` cannot downgrade a real ``GeoDataFrame`` and + strip its geometry or CRS. When every frame is empty, the first frame is + returned to preserve its concrete type. + + When multiple non-empty frames are combined, non-null feature IDs are + deduplicated regardless of the plan axis. Filter clauses can match the same + feature, and list inputs can contain repeated values or otherwise select + overlapping records. Rows without an ``id`` are preserved verbatim: pandas + treats null values as duplicates, so deduplicating them would silently lose + data. + """ + non_empty = [frame for frame in frames if not frame.empty] + if not non_empty: + return frames[0] if frames else pd.DataFrame() + if len(non_empty) == 1: + return non_empty[0].copy() + + combined = pd.concat(non_empty, ignore_index=True) + if "id" not in combined.columns: + return combined + + has_id = combined["id"].notna() + if has_id.all(): + return combined.drop_duplicates(subset="id", ignore_index=True) + if has_id.any(): + id_rows = combined[has_id].drop_duplicates(subset="id") + no_id_rows = combined[~has_id] + return pd.concat([id_rows, no_id_rows], ignore_index=True) + return combined + + +def _combine_chunk_responses( + responses: list[httpx.Response], canonical_url: str | None +) -> httpx.Response: + """ + Fold per-sub-request responses into a single aggregated response. + + For a multi-response input, returns a shallow copy of + ``responses[0]`` with ``.headers`` set to those of the response reporting + the lowest ``x-ratelimit-remaining`` value (the most conservative quota + observation; see :func:`_lowest_remaining`), ``.elapsed`` set to the sum of + the per-response elapsed durations, and ``.url`` set to the + canonical original-query URL (when supplied) so ``BaseMetadata`` + reflects the user's full request rather than the first chunk. + + For a single-response input with no canonical-URL override, + ``responses[0]`` is returned unchanged to skip the copy on the + passthrough hot path. + + Parameters + ---------- + responses : list[httpx.Response] + One response per completed sub-request, in caller-provided order. + canonical_url : str or None + URL of the unchunked original request. ``None`` skips the URL + override — used by the passthrough path (the fetcher's + response already carries the original-query URL) and by the + worst-case overflow path (no buildable canonical URL exists). + + Returns + ------- + httpx.Response + A shallow copy of the first response with aggregated + ``headers``, ``elapsed``, and ``url``. The function is + idempotent (the input responses' ``headers`` / ``elapsed`` / + ``url`` are never mutated), so it's safe to call repeatedly + via :attr:`ChunkedCall.partial_response` during error + inspection or resume retries. ``headers`` on the returned + object is a fresh ``httpx.Headers``, so mutations there don't + back-propagate into any chunk's underlying response. + """ + if len(responses) == 1 and canonical_url is None: + return responses[0] + + # Headers come from the response with the lowest reported remaining quota; + # ``_lowest_remaining`` returns the lone response as-is + # for a single-element list). ``_merge_response`` re-sums elapsed onto a + # fresh copy, so repeated calls (e.g. via ``ChunkedCall.partial_response`` + # during resume) stay idempotent. + elapsed = sum((_safe_elapsed(r) for r in responses), start=timedelta()) + return _merge_response( + responses[0], + headers_from=_lowest_remaining(responses), + elapsed=elapsed, + url=canonical_url, + ) diff --git a/dataretrieval/transport/http.py b/dataretrieval/transport/http.py new file mode 100644 index 00000000..f8643de8 --- /dev/null +++ b/dataretrieval/transport/http.py @@ -0,0 +1,93 @@ +"""HTTP client lifecycle, timeout defaults, and host-scoped authentication.""" + +from __future__ import annotations + +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _pkg_version +from typing import Any + +import httpx + +from dataretrieval.exceptions import NetworkError + +try: + _PACKAGE_VERSION = _pkg_version("dataretrieval") +except PackageNotFoundError: + _PACKAGE_VERSION = "version-unknown" + +USER_AGENT = f"python-dataretrieval/{_PACKAGE_VERSION}" + +HTTPX_DEFAULTS: dict[str, Any] = { + "follow_redirects": True, + "timeout": httpx.Timeout(60.0, connect=10.0), +} + +_AUTHORIZED_API_KEY_HOST = "api.waterdata.usgs.gov" + + +def default_headers(target_url: str | httpx.URL | None = None) -> dict[str, str]: + """Build standard headers, scoping ``API_USGS_PAT`` to its authorized host.""" + headers = { + "Accept-Encoding": "compress, gzip", + "Accept": "application/json", + "User-Agent": USER_AGENT, + "lang": "en-US", + } + token = os.getenv("API_USGS_PAT") + if token and target_url is not None: + try: + host = httpx.URL(str(target_url)).host + except (httpx.InvalidURL, TypeError): + host = None + if host == _AUTHORIZED_API_KEY_HOST: + headers["X-Api-Key"] = token + return headers + + +def strip_api_key_from_untrusted_host(request: httpx.Request) -> None: + """Remove Water Data credentials before sending to any other host.""" + if request.url.host != _AUTHORIZED_API_KEY_HOST: + request.headers.pop("X-Api-Key", None) + + +async def strip_api_key_from_untrusted_host_async(request: httpx.Request) -> None: + """Async-client form of :func:`strip_api_key_from_untrusted_host`.""" + strip_api_key_from_untrusted_host(request) + + +HTTPX_ASYNC_DEFAULTS: dict[str, Any] = { + **HTTPX_DEFAULTS, + "event_hooks": {"request": [strip_api_key_from_untrusted_host_async]}, +} + + +def network_error(url: str | httpx.URL, exc: httpx.TransportError) -> NetworkError: + """Build a typed error for a failed round trip with no HTTP response.""" + detail = str(exc) or type(exc).__name__ + return NetworkError(f"Could not reach the service at {url}: {detail}") + + +def get(url: str | httpx.URL, **kwargs: Any) -> httpx.Response: + """Issue one guarded synchronous GET and map transport failures.""" + client_options: dict[str, Any] = { + key: kwargs.pop(key) + for key in ("follow_redirects", "timeout", "transport", "verify") + if key in kwargs + } + client_options["event_hooks"] = {"request": [strip_api_key_from_untrusted_host]} + try: + with httpx.Client(**client_options) as client: + return client.get(url, **kwargs) + except httpx.TransportError as exc: + raise network_error(url, exc) from exc + + +@asynccontextmanager +async def open_async_client(**overrides: Any) -> AsyncIterator[httpx.AsyncClient]: + """Open a short-lived async client with redirect-safe shared defaults.""" + options = {**HTTPX_ASYNC_DEFAULTS, **overrides} + async with httpx.AsyncClient(**options) as client: + yield client diff --git a/dataretrieval/transport/pagination.py b/dataretrieval/transport/pagination.py new file mode 100644 index 00000000..7c38e1b8 --- /dev/null +++ b/dataretrieval/transport/pagination.py @@ -0,0 +1,129 @@ +"""Callback-driven cursor pagination independent of any service protocol.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from typing import Any, TypeVar + +import httpx +import pandas as pd + +from dataretrieval.exceptions import DataRetrievalError, RateLimited +from dataretrieval.transport import progress as _progress +from dataretrieval.transport.combining import ( + _QUOTA_HEADER, + _merge_response, + _safe_elapsed, +) +from dataretrieval.transport.http import open_async_client + +logger = logging.getLogger(__name__) +_Cursor = TypeVar("_Cursor") + + +@asynccontextmanager +async def _client_for( + client: httpx.AsyncClient | None, +) -> AsyncIterator[httpx.AsyncClient]: + """Borrow a caller client or open a guarded short-lived client.""" + if client is not None: + yield client + return + async with open_async_client() as new: + yield new + + +def paginated_failure_message(pages_collected: int, cause: BaseException) -> str: + """Build a recovery-oriented message for an interrupted page walk.""" + cause_str = str(cause).removesuffix(".") + if not cause_str.strip(): + cause_str = type(cause).__name__ + if isinstance(cause, RateLimited): + action = "wait for the rate-limit window to reset and retry" + else: + action = "retry the request (possibly after a short backoff)" + return ( + f"Paginated request failed after collecting {pages_collected} " + f"page(s): {cause_str}. To recover: {action}, reduce the " + f"request size (e.g. fewer locations, a shorter time range, or " + f"a smaller ``limit``), or obtain an API token." + ) + + +async def paginate( + initial_req: httpx.Request, + *, + parse_response: Callable[[httpx.Response], tuple[pd.DataFrame, _Cursor | None]], + follow_up: Callable[[_Cursor, httpx.AsyncClient], Awaitable[httpx.Response]], + raise_for_status: Callable[[httpx.Response], None], + client: httpx.AsyncClient | None = None, + row_cap: int | None = None, +) -> tuple[pd.DataFrame, httpx.Response]: + """Fetch and combine pages until the injected parser returns no cursor. + + The service adapter supplies response parsing, cursor following, and status + mapping. This loop owns client lifecycle, repeated-cursor protection, + optional row capping, progress updates, failure wrapping, and response + metadata aggregation. + """ + logger.debug("Requesting: %s", initial_req.url) + reporter = _progress.current() + + def report_page(page: httpx.Response, frame: pd.DataFrame) -> None: + if reporter is not None: + reporter.set_rate_remaining( + page.headers.get(_QUOTA_HEADER), + limit=page.headers.get("x-ratelimit-limit"), + ) + reporter.add_page(rows=len(frame)) + + async with _client_for(client) as session: + response = await session.send(initial_req) + raise_for_status(response) + initial_response = response + total_elapsed = _safe_elapsed(response) + + try: + frame, cursor = parse_response(response) + except Exception as exc: # noqa: BLE001 + logger.warning("Initial response parse failed.") + raise DataRetrievalError(paginated_failure_message(0, exc)) from exc + + frames = [frame] + nrows = len(frame) + seen: set[Any] = set() + report_page(response, frame) + + while ( + cursor is not None + and cursor not in seen + and (row_cap is None or nrows < row_cap) + ): + seen.add(cursor) + try: + response = await follow_up(cursor, session) + raise_for_status(response) + frame, cursor = parse_response(response) + frames.append(frame) + nrows += len(frame) + total_elapsed += _safe_elapsed(response) + report_page(response, frame) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Request failed at cursor %r. Data download interrupted.", cursor + ) + raise DataRetrievalError( + paginated_failure_message(len(frames), exc) + ) from exc + + final_response = _merge_response( + initial_response, + headers_from=response, + elapsed=total_elapsed, + ) + result = pd.concat(frames, ignore_index=True) + if row_cap is not None: + result = result.head(row_cap) + return result, final_response diff --git a/dataretrieval/transport/progress.py b/dataretrieval/transport/progress.py new file mode 100644 index 00000000..7c6ef133 --- /dev/null +++ b/dataretrieval/transport/progress.py @@ -0,0 +1,294 @@ +"""A single self-updating status line for paginated and chunked queries. + +Retrieval adapters can fan out in ways the caller cannot see: large multi-value +requests are split into URL-length-safe *chunks* (``chunking`` module), and each +request follows ``next`` links across an unknown number of *pages* +(``transport.pagination.paginate``). This module surfaces that work as one +line on stderr, +rewritten in place as data arrives:: + + Retrieving: daily · 6 pages · 2,881 rows · 995/1,000 requests remaining + +The active reporter lives in a :class:`~contextvars.ContextVar` rather than being +threaded through every signature: progress is a cross-cutting concern that the +chunk orchestrator (outer, chunk counts) and the page-walking loop (inner, +page/row/rate-limit counts) both update without knowing about each other. Call +:func:`progress_context` to activate one and :func:`current` to reach it. + +By default the line is shown for interactive use — an interactive terminal or a +Jupyter/IPython kernel, like ``tqdm`` — while redirected logs and CI stay clean. +``API_USGS_PROGRESS`` forces it on (``1``/``true``) or off (``0``/``false``). +""" + +from __future__ import annotations + +import contextvars +import os +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from typing import TextIO + + +def _group_int(value: str) -> str: + """Comma-group a plain ASCII integer string; pass anything else through. + + (``str.isdigit`` alone is True for non-decimal unicode digits that ``int`` + rejects, hence the ``isascii`` guard.) + """ + return f"{int(value):,}" if value.isascii() and value.isdigit() else value + + +# The reporter active for the current query. A ContextVar (not a module global) +# so the chunk orchestrator and the page loop resolve to the same reporter +# within one query, and an unrelated query in another context can't clobber its +# state. (It does not give concurrent queries sharing one stderr separate +# lines — they would still interleave.) +_active: contextvars.ContextVar[ProgressReporter | None] = contextvars.ContextVar( + "transport_progress", default=None +) + +# Where to register for an API key. Surfaced once when a query runs without an +# API key configured (no API_USGS_PAT), since unauthenticated callers hit much +# lower rate limits (see the API_USGS_PAT note in the README). +SIGNUP_URL = "https://api.waterdata.usgs.gov/signup/" + +# Process-level latch so the "no API key" pointer is shown at most once. +_api_key_hint_shown = False + + +def _in_jupyter_kernel() -> bool: + """True when running inside a Jupyter/IPython *kernel* (notebook, lab, + qtconsole). + + A kernel's ``stderr`` isn't a TTY, but it honors carriage-return rewrites in + the cell output area — the same mechanism ``tqdm`` rides on — so the line is + worth showing there. The plain IPython terminal REPL is a + ``TerminalInteractiveShell`` (already a TTY), so only the ZMQ kernel needs + this extra signal. Detected without importing IPython: if it isn't already + imported, we aren't in a shell. + """ + ipython = sys.modules.get("IPython") + if ipython is None: + return False + shell = ipython.get_ipython() + return shell is not None and type(shell).__name__ == "ZMQInteractiveShell" + + +def _enabled_default(stream: TextIO) -> bool: + """Whether to draw the line by default. + + ``API_USGS_PROGRESS`` wins when set. Otherwise show it for interactive use — + a TTY or a Jupyter/IPython kernel — and stay quiet for redirected output, + logs, and CI. + """ + override = os.getenv("API_USGS_PROGRESS") + if override is not None: + return override.strip().lower() not in {"", "0", "false", "no", "off"} + if _in_jupyter_kernel(): + return True + return hasattr(stream, "isatty") and stream.isatty() + + +class ProgressReporter: + """Accumulates query progress and rewrites a single status line in place. + + Every update method is a no-op when the reporter is disabled, so call sites + need no ``if enabled`` guards. The line is redrawn with a leading carriage + return and padded to erase the previous (possibly longer) contents; + :meth:`close` terminates it with a newline so the final state persists. + """ + + def __init__( + self, + *, + service: str | None = None, + stream: TextIO | None = None, + enabled: bool | None = None, + ) -> None: + self._stream = stream if stream is not None else sys.stderr + self.enabled = _enabled_default(self._stream) if enabled is None else enabled + # The service/collection being retrieved (e.g. "daily", "peaks"), + # shown as the line's leading label. + self.service = service + self.total_chunks = 1 + self.current_chunk = 0 + self.pages = 0 + self.rows = 0 + self.rate_remaining: str | None = None + # The hourly request quota (``x-ratelimit-limit``), shown as the + # denominator when the server reports it. + self.rate_limit: str | None = None + # Transient note shown while a sub-request backs off before a + # retry; cleared by the next page/chunk so it doesn't linger. + self.retry_note: str | None = None + self._last_len = 0 + # Whether anything was actually written to the stream — drives whether + # close() needs a terminating newline. (``current_chunk`` is a poor + # proxy: ``start_chunk`` sets it even when it doesn't render.) + self._rendered = False + self._closed = False + + def set_chunks(self, total: int) -> None: + """Record how many filter chunks this query was split into.""" + self.total_chunks = max(int(total), 1) + + def start_chunk(self, index: int) -> None: + """Mark the start of chunk ``index`` (1-based) and redraw. + + Only redraws when actually chunking (``total_chunks > 1``); a + single-chunk plan has nothing chunk-specific to show yet, so it + avoids a premature "0 pages" frame before the first page arrives. + """ + self.current_chunk = index + self.retry_note = None + if self.total_chunks > 1: + self._render() + + def add_page(self, rows: int = 0) -> None: + """Record one fetched page carrying ``rows`` rows and redraw.""" + self.pages += 1 + self.rows += int(rows) + self.retry_note = None + self._render() + + def note_retry(self, *, attempt: int, wait: float) -> None: + """Show that a sub-request is backing off before retry ``attempt``. + + Cleared by the next :meth:`add_page` / :meth:`start_chunk` (or by + :meth:`close`) so the line returns to normal once the retry resolves. + """ + # Keep sub-second waits explicit (avoid misleading ``0s``) while + # rendering whole-second waits without unnecessary ``.0`` noise. + # ``float()`` to support Python 3.9-3.11: ``round(int, 1)`` returns an + # int and ``int.is_integer()`` (used below) only exists on 3.12+. + wait_1dp = round(float(wait), 1) + if wait_1dp < 1 or not wait_1dp.is_integer(): + secs = f"{wait_1dp:.1f}s" + else: + secs = f"{wait_1dp:.0f}s" + self.retry_note = f"retrying (attempt {attempt}, waiting {secs})" + self._render() + + def set_rate_remaining( + self, value: str | int | None, limit: str | int | None = None + ) -> None: + """Update the rate-limit display from the response headers. + + ``value`` is ``x-ratelimit-remaining``; ``limit`` is the optional + ``x-ratelimit-limit`` quota, shown as the denominator. Empty/missing + values are ignored so a page that omits a header doesn't blank out the + last known value. + """ + if value not in (None, ""): + self.rate_remaining = str(value) + if limit not in (None, ""): + self.rate_limit = str(limit) + + def _format(self) -> str: + parts: list[str] = [] + if self.total_chunks > 1: + parts.append(f"chunk {self.current_chunk}/{self.total_chunks}") + parts.append(f"{self.pages} page" + ("" if self.pages == 1 else "s")) + if self.rows: + parts.append(f"{self.rows:,} rows") + if self.rate_remaining is not None: + remaining = _group_int(self.rate_remaining) + if self.rate_limit is not None: + limit = _group_int(self.rate_limit) + segment = f"{remaining}/{limit} requests remaining" + else: + segment = f"{remaining} requests remaining" + parts.append(segment) + if self.retry_note is not None: + parts.append(self.retry_note) + if self.service: + return f"Retrieving: {self.service} · " + " · ".join(parts) + return "Progress: " + " · ".join(parts) + + def _render(self) -> None: + if not self.enabled or self._closed: + return + try: + line = self._format() + pad = max(self._last_len - len(line), 0) + self._stream.write("\r" + line + " " * pad) + self._stream.flush() + self._last_len = len(line) + self._rendered = True + except Exception: # noqa: BLE001 + # Progress output is best-effort cosmetics; a broken pipe (output + # piped to ``head``), a closed stream, or an encoding error must + # never disturb — let alone truncate — the query. Disable so we + # don't retry on every subsequent page. + self.enabled = False + + def close(self) -> None: + """Finalize the line with a trailing newline so it persists on screen. + + If no API key is configured (no ``API_USGS_PAT``), append a one-time + pointer to API-key registration, since unauthenticated callers hit much + lower rate limits. + """ + if self._closed: + return + # A retry note set during the final backoff would otherwise freeze as + # the persisted last line of a call that has since completed or given + # up; clear it and redraw (while still un-closed, so ``_render`` runs) + # so the final state isn't a stale "retrying". + if self.enabled and self._rendered and self.retry_note is not None: + self.retry_note = None + self._render() + self._closed = True + if not (self.enabled and self._rendered): + return + try: + self._stream.write("\n") + self._maybe_hint_api_key() + self._stream.flush() + except Exception: # noqa: BLE001 + self.enabled = False + + def _maybe_hint_api_key(self) -> None: + global _api_key_hint_shown + if _api_key_hint_shown or os.getenv("API_USGS_PAT"): + return + # Set the once-per-process latch only after a successful write, so a + # failed write (broken pipe) doesn't silently burn the hint for every + # later query in the process. + self._stream.write( + f"No API key detected — register for higher rate limits at {SIGNUP_URL}\n" + ) + _api_key_hint_shown = True + + +@contextmanager +def progress_context( + *, + service: str | None = None, + stream: TextIO | None = None, + enabled: bool | None = None, +) -> Iterator[ProgressReporter]: + """Activate a :class:`ProgressReporter` for the duration of a query. + + ``service`` labels the line (e.g. ``"Retrieving: daily ..."``). If a reporter + is already active (a nested call), the existing one is yielded unchanged so + the outermost query owns the single line; only the outermost context closes + it (and ``service``/``stream``/``enabled`` of a nested call are ignored). + """ + existing = _active.get() + if existing is not None: + yield existing + return + reporter = ProgressReporter(service=service, stream=stream, enabled=enabled) + token = _active.set(reporter) + try: + yield reporter + finally: + _active.reset(token) + reporter.close() + + +def current() -> ProgressReporter | None: + """Return the reporter active for the current query, or ``None``.""" + return _active.get() diff --git a/dataretrieval/transport/retry.py b/dataretrieval/transport/retry.py new file mode 100644 index 00000000..339c36f3 --- /dev/null +++ b/dataretrieval/transport/retry.py @@ -0,0 +1,160 @@ +"""Bounded retry policy and transient-failure classification.""" + +from __future__ import annotations + +import asyncio +import os +import random +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from typing import TypeVar + +import httpx + +from dataretrieval.exceptions import NetworkError, TransientError +from dataretrieval.transport import progress as _progress + +_RETRIES_ENV = "API_USGS_RETRIES" +_RETRIES_DEFAULT = 4 +_RETRY_BASE_BACKOFF = 0.5 +_RETRY_MAX_BACKOFF = 30.0 +_RETRY_AFTER_CAP = 60.0 + +_T = TypeVar("_T") + + +def parse_retry_after(value: str | None) -> float | None: + """Parse ``Retry-After`` delta-seconds or HTTP-date into seconds.""" + if not value: + return None + raw = value.strip() + try: + return max(0.0, float(raw)) + except ValueError: + pass + try: + retry_at = parsedate_to_datetime(raw) + except (TypeError, ValueError, OverflowError): + return None + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + return max(0.0, (retry_at - datetime.now(timezone.utc)).total_seconds()) + + +def _read_retries_env() -> int: + """Resolve ``API_USGS_RETRIES`` to retries after the first attempt.""" + raw = os.environ.get(_RETRIES_ENV) + if raw is None or raw.strip() == "": + return _RETRIES_DEFAULT + try: + value = int(raw.strip()) + except ValueError as exc: + raise ValueError( + f"{_RETRIES_ENV} must be a non-negative integer (got {raw!r})." + ) from exc + if value < 0: + raise ValueError(f"{_RETRIES_ENV} must be >= 0 (got {value}).") + return value + + +@dataclass(frozen=True) +class RetryPolicy: + """Immutable bounded exponential-backoff-with-full-jitter policy.""" + + max_retries: int = _RETRIES_DEFAULT + base_backoff: float = _RETRY_BASE_BACKOFF + max_backoff: float = _RETRY_MAX_BACKOFF + retry_after_cap: float = _RETRY_AFTER_CAP + + def __post_init__(self) -> None: + if self.max_retries < 0: + raise ValueError(f"max_retries must be >= 0 (got {self.max_retries}).") + if self.base_backoff < 0 or self.max_backoff < 0 or self.retry_after_cap < 0: + raise ValueError("retry backoff settings must be non-negative.") + + @classmethod + def from_env(cls) -> RetryPolicy: + """Build a policy from current environment and module defaults.""" + return cls( + max_retries=_read_retries_env(), + base_backoff=_RETRY_BASE_BACKOFF, + max_backoff=_RETRY_MAX_BACKOFF, + retry_after_cap=_RETRY_AFTER_CAP, + ) + + def should_retry(self, attempt: int, retry_after: float | None) -> bool: + """Whether a just-failed 1-based attempt warrants another try.""" + if attempt > self.max_retries: + return False + return retry_after is None or retry_after <= self.retry_after_cap + + def backoff(self, attempt: int, retry_after: float | None) -> float: + """Seconds to wait before a 1-based retry attempt.""" + if retry_after is not None: + return retry_after + ceiling = min(self.max_backoff, self.base_backoff * 2 ** (attempt - 1)) + return random.uniform(0.0, ceiling) + + +_NO_RETRY = RetryPolicy(max_retries=0) + + +def _retryable(exc: BaseException) -> tuple[bool, float | None]: + """Return whether ``exc`` is safe to retry and any server delay hint.""" + if isinstance(exc, TransientError): + return True, exc.retry_after + if isinstance(exc, (NetworkError, httpx.TransportError)): + return True, None + return False, None + + +def _retry_delay(exc: BaseException, attempt: int, policy: RetryPolicy) -> float | None: + """Return the bounded delay for a failed attempt, or ``None`` to stop.""" + retryable, retry_after = _retryable(exc) + if not retryable or not policy.should_retry(attempt, retry_after): + return None + delay = policy.backoff(attempt, retry_after) + reporter = _progress.current() + if reporter is not None: + reporter.note_retry(attempt=attempt, wait=delay) + return delay + + +async def retry_async( + afn: Callable[[], Awaitable[_T]], policy: RetryPolicy | None = None +) -> _T: + """Call an awaitable with bounded retry on typed transient failures.""" + policy = RetryPolicy.from_env() if policy is None else policy + attempt = 0 + while True: + try: + return await afn() + except Exception as exc: # noqa: BLE001 - re-raised unless retryable + attempt += 1 + delay = _retry_delay(exc, attempt, policy) + if delay is None: + raise + await asyncio.sleep(delay) + + +def retry_sync(fn: Callable[[], _T], policy: RetryPolicy | None = None) -> _T: + """Call a synchronous operation with bounded retry on typed transients. + + ``KeyboardInterrupt``, ``SystemExit``, and other cancellation signals are + not caught because the loop handles ``Exception`` rather than + ``BaseException``. + """ + policy = RetryPolicy.from_env() if policy is None else policy + attempt = 0 + while True: + try: + return fn() + except Exception as exc: # noqa: BLE001 - re-raised unless retryable + attempt += 1 + delay = _retry_delay(exc, attempt, policy) + if delay is None: + raise + time.sleep(delay) diff --git a/dataretrieval/transport/sync.py b/dataretrieval/transport/sync.py new file mode 100644 index 00000000..8b4f19f2 --- /dev/null +++ b/dataretrieval/transport/sync.py @@ -0,0 +1,29 @@ +"""Synchronous dispatch over asynchronous retrieval internals.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import TypeVar, cast + +import httpx +from anyio.from_thread import start_blocking_portal + +from dataretrieval.transport import progress as _progress +from dataretrieval.transport.http import network_error + +_T = TypeVar("_T") + + +def run_sync( + make_coro: Callable[[], Awaitable[_T]], + *, + service: str, + error_url: str | httpx.URL, +) -> _T: + """Run an async retrieval from synchronous code in a blocking portal.""" + with _progress.progress_context(service=service): + with start_blocking_portal() as portal: + try: + return cast("_T", portal.call(make_coro)) + except httpx.TransportError as exc: + raise network_error(error_url, exc) from exc diff --git a/dataretrieval/utils.py b/dataretrieval/utils.py index 7506a469..32a68fa2 100644 --- a/dataretrieval/utils.py +++ b/dataretrieval/utils.py @@ -5,38 +5,40 @@ from __future__ import annotations import numbers -import os import warnings from collections.abc import Callable, Iterable, Iterator from contextlib import contextmanager from contextvars import ContextVar -from importlib.metadata import PackageNotFoundError -from importlib.metadata import version as _pkg_version from typing import Any, Generic, TypeVar import httpx import pandas as pd +import dataretrieval.transport.http as _transport_http from dataretrieval.codes import tz from dataretrieval.exceptions import ( - NetworkError, NoSitesError, URLTooLong, error_for_status, ) +from dataretrieval.transport.retry import ( + RetryPolicy, + parse_retry_after, + retry_sync, +) -try: - _PACKAGE_VERSION = _pkg_version("dataretrieval") -except PackageNotFoundError: - _PACKAGE_VERSION = "version-unknown" - -# Typed as ``dict[str, Any]`` (not the inferred ``dict[str, object]``) so that -# splatting it as ``**HTTPX_DEFAULTS`` into ``httpx.get`` / ``httpx.AsyncClient`` -# type-checks: the values are a heterogeneous bag of httpx keyword arguments. -HTTPX_DEFAULTS: dict[str, Any] = { - "follow_redirects": True, - "timeout": httpx.Timeout(60.0, connect=10.0), -} +# Compatibility names retained at their historical utility paths. +_AUTHORIZED_API_KEY_HOST = _transport_http._AUTHORIZED_API_KEY_HOST +HTTPX_ASYNC_DEFAULTS = _transport_http.HTTPX_ASYNC_DEFAULTS +HTTPX_DEFAULTS = _transport_http.HTTPX_DEFAULTS +USER_AGENT = _transport_http.USER_AGENT +_default_headers = _transport_http.default_headers +_get = _transport_http.get +_network_error = _transport_http.network_error +_strip_api_key_from_untrusted_host = _transport_http.strip_api_key_from_untrusted_host +_strip_api_key_from_untrusted_host_async = ( + _transport_http.strip_api_key_from_untrusted_host_async +) _T = TypeVar("_T") @@ -103,75 +105,6 @@ def _require_positive_int( raise ValueError(f"{name} must be a positive integer{eg} (got {value!r}).") -# The single authorized host for the API key. The key is a USGS personal -# access token issued for the Water Data API and must never be forwarded to -# non-USGS hosts, lookalikes, or external endpoints (including STAC rating -# asset downloads). -_AUTHORIZED_API_KEY_HOST = "api.waterdata.usgs.gov" - - -def _default_headers(target_url: str | httpx.URL | None = None) -> dict[str, str]: - """Build the default HTTP headers for a USGS web-API request. - - Always sets a descriptive ``User-Agent`` plus ``Accept`` / - ``Accept-Encoding`` and ``lang``. If the ``API_USGS_PAT`` environment - variable is set AND ``target_url`` points to the explicitly authorized - Water Data host (``api.waterdata.usgs.gov``), its value is added as the - ``X-Api-Key`` header. The key is never sent to other hosts. - - Parameters - ---------- - target_url : str or httpx.URL or None - The URL the request will be sent to. When provided, the API key is - included only if the host matches the authorized Water Data host. - When ``None`` (legacy callers), the key is NOT included — callers - must pass the concrete URL. - - Returns - ------- - dict[str, str] - Headers suitable for an ``httpx`` request. - """ - headers = { - "Accept-Encoding": "compress, gzip", - "Accept": "application/json", - "User-Agent": f"python-dataretrieval/{_PACKAGE_VERSION}", - "lang": "en-US", - } - token = os.getenv("API_USGS_PAT") - if token and target_url is not None: - try: - host = httpx.URL(str(target_url)).host - except (httpx.InvalidURL, TypeError): - host = None - if host == _AUTHORIZED_API_KEY_HOST: - headers["X-Api-Key"] = token - return headers - - -def _strip_api_key_from_untrusted_host(request: httpx.Request) -> None: - """Remove Water Data credentials from any request to another host. - - HTTPX retains arbitrary custom headers across cross-origin redirects. This - hook runs for the initial request and every redirect, making host scoping an - execution-time invariant rather than relying only on initial header - construction. - """ - if request.url.host != _AUTHORIZED_API_KEY_HOST: - request.headers.pop("X-Api-Key", None) - - -async def _strip_api_key_from_untrusted_host_async(request: httpx.Request) -> None: - """Async-client form of :func:`_strip_api_key_from_untrusted_host`.""" - _strip_api_key_from_untrusted_host(request) - - -HTTPX_ASYNC_DEFAULTS: dict[str, Any] = { - **HTTPX_DEFAULTS, - "event_hooks": {"request": [_strip_api_key_from_untrusted_host_async]}, -} - - def to_str(listlike: object, delimiter: str = ",") -> str | None: """Translates list-like objects into strings. @@ -427,35 +360,6 @@ def _url_too_long_error(detail: str) -> URLTooLong: ) -def _network_error(url: str | httpx.URL, exc: httpx.TransportError) -> NetworkError: - """Build the :class:`~dataretrieval.exceptions.NetworkError` for a failed - round-trip ``exc`` (no HTTP response: timeout, DNS, refused connection).""" - # Some httpx transport errors stringify empty (e.g. ``ConnectTimeout()``); - # fall back to the class name so the message is always informative. - detail = str(exc) or type(exc).__name__ - return NetworkError(f"Could not reach the service at {url}: {detail}") - - -def _get(url: str | httpx.URL, **kwargs: Any) -> httpx.Response: - """Issue one guarded synchronous GET and map transport failures. - - A short-lived client supplies a request hook for both the initial request - and every redirect. The hook removes ``X-Api-Key`` unless the destination - host is the authorized Water Data API host. - """ - client_options: dict[str, Any] = { - key: kwargs.pop(key) - for key in ("follow_redirects", "timeout", "transport", "verify") - if key in kwargs - } - client_options["event_hooks"] = {"request": [_strip_api_key_from_untrusted_host]} - try: - with httpx.Client(**client_options) as client: - return client.get(url, **kwargs) - except httpx.TransportError as exc: - raise _network_error(url, exc) from exc - - def _raise_for_status( response: httpx.Response, *, @@ -487,14 +391,40 @@ def _raise_for_status( if detail: message += f": {detail}" message += f" (URL: {response.url})" - raise error_for_status(status, message) + raise error_for_status( + status, + message, + retry_after=parse_retry_after(response.headers.get("Retry-After")), + ) -def query( +def _get_with_retry( + url: str | httpx.URL, + *, + detail_from: Callable[[httpx.Response], str | None] | None = None, + retry_policy: RetryPolicy | None = None, + **kwargs: Any, +) -> httpx.Response: + """GET with status mapping and bounded retry on typed transients.""" + + def attempt() -> httpx.Response: + response = _get(url, **kwargs) + _raise_for_status(response, detail_from=detail_from) + return response + + try: + return retry_sync(attempt, retry_policy) + except httpx.InvalidURL as exc: + raise _url_too_long_error(f"httpx rejected the URL client-side: {exc}") from exc + + +def _query_impl( url: str, payload: dict[str, Any], delimiter: str = ",", ssl_check: bool = True, + *, + retry_policy: RetryPolicy, ) -> httpx.Response: """Send a query. @@ -535,20 +465,16 @@ def query( # Drop them. (``to_str`` returns None for non-iterable scalars like bools.) payload = {k: v for k, v in payload.items() if v is not None} - user_agent = {"user-agent": f"python-dataretrieval/{_PACKAGE_VERSION}"} + user_agent = {"user-agent": USER_AGENT} - try: - response = _get( - url, - params=payload, - headers=user_agent, - verify=ssl_check, - **HTTPX_DEFAULTS, - ) - except httpx.InvalidURL as exc: - raise _url_too_long_error(f"httpx rejected the URL client-side: {exc}") from exc - - _raise_for_status(response) + response = _get_with_retry( + url, + params=payload, + headers=user_agent, + verify=ssl_check, + retry_policy=retry_policy, + **HTTPX_DEFAULTS, + ) # USGS waterservices signals an empty result with a 200 whose body starts # "No sites/data ..." (its legacy wording); surface it as NoSitesError. @@ -556,3 +482,37 @@ def query( raise NoSitesError(response.url) return response + + +def query( + url: str, + payload: dict[str, Any], + delimiter: str = ",", + ssl_check: bool = True, +) -> httpx.Response: + return _query_impl( + url, + payload, + delimiter, + ssl_check, + retry_policy=RetryPolicy(max_retries=0), + ) + + +query.__doc__ = _query_impl.__doc__ + + +def _query_with_retry( + url: str, + payload: dict[str, Any], + delimiter: str = ",", + ssl_check: bool = True, +) -> httpx.Response: + """Active-service form of :func:`query` with bounded transient retry.""" + return _query_impl( + url, + payload, + delimiter, + ssl_check, + retry_policy=RetryPolicy.from_env(), + ) diff --git a/dataretrieval/waterdata/api.py b/dataretrieval/waterdata/api.py index 7073d183..71fe61af 100644 --- a/dataretrieval/waterdata/api.py +++ b/dataretrieval/waterdata/api.py @@ -1,3439 +1,65 @@ -"""Functions for downloading data from the Water Data APIs, including the USGS -Aquarius Samples database. - -See https://api.waterdata.usgs.gov/ for API reference. -""" +"""Backward-compatible facade for Water Data collection-family adapters.""" from __future__ import annotations -import json -import logging -from collections.abc import Iterable -from io import StringIO -from typing import Any, get_args -from urllib.parse import quote - -import httpx -import pandas as pd - -from dataretrieval.ogc import fetch_ogc_request -from dataretrieval.ogc.errors import _raise_for_non_200 -from dataretrieval.ogc.filters import FILTER_LANG -from dataretrieval.ogc.requests import ( - _as_str_list, - _check_ogc_requests, - _construct_cql_request, - _switch_properties_id, +from dataretrieval.waterdata import samples as _samples +from dataretrieval.waterdata.cql import get_cql +from dataretrieval.waterdata.measurements import ( + get_channel, + get_field_measurements, + get_peaks, ) -from dataretrieval.utils import ( - HTTPX_DEFAULTS, - BaseMetadata, - _attach_datetime_columns, - _default_headers, - _get, - to_str, +from dataretrieval.waterdata.metadata import ( + get_combined_metadata, + get_field_measurements_metadata, + get_monitoring_locations, + get_time_series_metadata, ) -from dataretrieval.waterdata import stats -from dataretrieval.waterdata.types import ( - CODE_SERVICES, - METADATA_COLLECTIONS, - PROFILES, - SERVICES, - WATERDATA_SERVICES, +from dataretrieval.waterdata.reference import get_queryables, get_reference_table +from dataretrieval.waterdata.samples import ( + get_codes, + get_samples, + get_samples_summary, ) -from dataretrieval.waterdata.utils import ( - _OUTPUT_ID_BY_SERVICE, - SAMPLES_URL, - _accept_legacy_kwargs, - _check_profiles, - _finalize_ogc, - _get_args, - _with_state, - get_ogc_data, +from dataretrieval.waterdata.time_series import ( + get_continuous, + get_daily, + get_latest_continuous, + get_latest_daily, + get_stats_date_range, + get_stats_por, ) - -# Set up logger for this module -logger = logging.getLogger(__name__) - - -def get_daily( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - daily_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data provide one data value to represent water conditions for the - day. - - Throughout much of the history of the USGS, the primary water data available - was daily data collected manually at the monitoring location once each day. - With improved availability of computer storage and automated transmission of - data, the daily data published today are generally a statistical summary or - metric of the continuous data collected each day, such as the daily mean, - minimum, or maximum value. Daily data are automatically calculated from the - continuous data of the same parameter code and are described by parameter - code and a statistic code. These data have also been referred to as “daily - values” or “DV”. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter - codes and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. - Available options are: geometry, id, time_series_id, - monitoring_location_id, parameter_code, statistic_id, time, value, - unit_of_measure, approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - daily_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - Only features that have a last_modified that intersects the value of - datetime are selected. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get daily flow data from a single site - >>> # over a yearlong period - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", - ... ) - - >>> # Quick "show me the last week" idiom (ISO 8601 duration) - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... time="P7D", - ... ) - - >>> # Get approved daily flow data from multiple sites - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], - ... approval_status="Approved", - ... time="2024-01-01/..", - ... ) - - >>> # Pull only rows whose underlying record was refreshed in the - >>> # last 7 days — handy for incremental ETL polling - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... last_modified="P7D", - ... ) - - >>> # Chain queries: pull all stream sites in a state, then their - >>> # daily discharge for the last week. The site list can be hundreds - >>> # of values long — the request is transparently chunked across - >>> # multiple sub-requests so the URL stays under the server's byte - >>> # limit. Combined output looks like a single query. - >>> sites_df, _ = dataretrieval.waterdata.get_monitoring_locations( - ... state="Ohio", - ... site_type="Stream", - ... ) - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id=sites_df["monitoring_location_id"].tolist(), - ... parameter_code="00060", - ... time="P7D", - ... ) - """ - service = "daily" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_continuous( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - continuous_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """ - Continuous data provide instantaneous water conditions. - - This is an early version of the continuous endpoint that is feature-complete - and is being made available for limited use. Geometries are not included - with the continuous endpoint. If the "time" input is left blank, the service - will return the most recent year of measurements. Users may request no more - than three years of data with each function call. - - Continuous data are collected at a high frequency, typically 15-minute - intervals. Depending on the specific monitoring location, the data may be - transmitted automatically via telemetry and be available on WDFN within - minutes of collection, while other times the delivery of data may be delayed - if the monitoring location does not have the capacity to automatically - transmit data. Continuous data are described by parameter name and - parameter code (pcode). These data might also be referred to as - "instantaneous values" or "IV". - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter - codes and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Continuous data are nearly always associated with statistic id - 00011. Using a different code (such as 00003 for mean) will - typically return no results. A complete list of codes and their - descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. - Available options are: geometry, id, time_series_id, - monitoring_location_id, parameter_code, statistic_id, time, value, - unit_of_measure, approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - continuous_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - Only features that have a last_modified that intersects the value of - datetime are selected. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 10000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get instantaneous gage height data from a - >>> # single site from a single year - >>> df, md = dataretrieval.waterdata.get_continuous( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00065", - ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", - ... ) - - >>> # Pull several disjoint time windows in one call via a CQL - >>> # ``filter``. See ``dataretrieval.ogc.filters`` for the - >>> # full grammar, auto-chunking, and pitfalls. - >>> df, md = dataretrieval.waterdata.get_continuous( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... filter=( - ... "(time >= '2023-06-01T12:00:00Z' " - ... "AND time <= '2023-06-01T13:00:00Z') " - ... "OR (time >= '2023-06-15T12:00:00Z' " - ... "AND time <= '2023-06-15T13:00:00Z')" - ... ), - ... filter_lang="cql-text", - ... ) - """ - service = "continuous" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_monitoring_locations( - monitoring_location_id: str | Iterable[str] | None = None, - agency_code: str | Iterable[str] | None = None, - agency_name: str | Iterable[str] | None = None, - monitoring_location_number: str | Iterable[str] | None = None, - monitoring_location_name: str | Iterable[str] | None = None, - district_code: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - country_name: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - state_name: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - county_name: str | Iterable[str] | None = None, - minor_civil_division_code: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type: str | Iterable[str] | None = None, - hydrologic_unit_code: str | Iterable[str] | None = None, - basin_code: str | Iterable[str] | None = None, - altitude: str | Iterable[str] | None = None, - altitude_accuracy: str | Iterable[str] | None = None, - altitude_method_code: str | Iterable[str] | None = None, - altitude_method_name: str | Iterable[str] | None = None, - vertical_datum: str | Iterable[str] | None = None, - vertical_datum_name: str | Iterable[str] | None = None, - horizontal_positional_accuracy_code: str | Iterable[str] | None = None, - horizontal_positional_accuracy: str | Iterable[str] | None = None, - horizontal_position_method_code: str | Iterable[str] | None = None, - horizontal_position_method_name: str | Iterable[str] | None = None, - original_horizontal_datum: str | Iterable[str] | None = None, - original_horizontal_datum_name: str | Iterable[str] | None = None, - drainage_area: str | Iterable[str] | None = None, - contributing_drainage_area: str | Iterable[str] | None = None, - time_zone_abbreviation: str | Iterable[str] | None = None, - uses_daylight_savings: str | Iterable[str] | None = None, - construction_date: str | Iterable[str] | None = None, - aquifer_code: str | Iterable[str] | None = None, - national_aquifer_code: str | Iterable[str] | None = None, - aquifer_type_code: str | Iterable[str] | None = None, - well_constructed_depth: str | Iterable[str] | None = None, - hole_constructed_depth: str | Iterable[str] | None = None, - depth_source_code: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Location information is basic information about the monitoring location - including the name, identifier, agency responsible for data collection, and - the date the location was established. It also includes information about - the type of location, such as stream, lake, or groundwater, and geographic - information about the location, such as state, county, latitude and - longitude, and hydrologic unit code (HUC). - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - agency_code : string or iterable of strings, optional - The agency that is reporting the data. Agency codes are fixed values - assigned by the National Water Information System (NWIS). - agency_name : string or iterable of strings, optional - The name of the agency that is reporting the data. - monitoring_location_number : string or iterable of strings, optional - Each monitoring location in the USGS data base has a unique 8- to - 15-digit identification number. Monitoring location numbers are - assigned based on this logic: - https://help.waterdata.usgs.gov/faq/sites/do-station-numbers-have-any-particular-meaning. - monitoring_location_name : string or iterable of strings, optional - This is the official name of the monitoring location in the database. - For well information this can be a district-assigned local number. - district_code : string or iterable of strings, optional - The Water Science Centers (WSCs) across the United States use the FIPS - state code as the district code. In some cases, monitoring locations and - samples may be managed by a water science center that is adjacent to the - state in which the monitoring location actually resides. For example a - monitoring location may have a district code of 30 which translates to - Montana, but the state code could be 56 for Wyoming because that is where - the monitoring location actually is located. - country_code : string or iterable of strings, optional - The code for the country in which the monitoring location is located. - country_name : string or iterable of strings, optional - The name of the country in which the monitoring location is located. - state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"``). - state_code : string or iterable of strings, optional - State code. A two-digit ANSI code (formerly FIPS code) as defined by - the American National Standards Institute, to define States and - equivalents. A three-digit ANSI code is used to define counties and - county equivalents. A `lookup table - `_ - is available. The only countries with - political subdivisions other than the US are Mexico and Canada. The Mexican - states have US state codes ranging from 81-86 and Canadian provinces have - state codes ranging from 90-98. - state_name : string or iterable of strings, optional - The name of the state or state equivalent in which the monitoring location - is located. - county_code : string or iterable of strings, optional - The code for the county or county equivalent (parish, borough, etc.) in which - the monitoring location is located. A `list of codes - `__ is available. - county_name : string or iterable of strings, optional - The name of the county or county equivalent (parish, borough, etc.) in which - the monitoring location is located. A `list of codes - `__ is available. - minor_civil_division_code : string or iterable of strings, optional - Codes for primary governmental or administrative divisions of the county or - county equivalent in which the monitoring location is located. - site_type_code : string or iterable of strings, optional - A code describing the hydrologic setting of the monitoring location. - site_type : string or iterable of strings, optional - A description of the hydrologic setting of the monitoring location. - hydrologic_unit_code : string or iterable of strings, optional - The United States is divided and sub-divided into successively smaller - hydrologic units which are classified into four levels: regions, - sub-regions, accounting units, and cataloging units. The hydrologic - units are arranged within each other, from the smallest (cataloging - units) to the largest (regions). Each hydrologic unit is identified by a - unique hydrologic unit code (HUC) consisting of two to eight digits - based on the four levels of classification in the hydrologic unit - system. - basin_code : string or iterable of strings, optional - The Basin Code or "drainage basin code" is a two-digit code that further - subdivides the 8-digit hydrologic-unit code. The drainage basin code is - defined by the USGS State Office where the monitoring location is - located. - altitude : string or iterable of strings, optional - Altitude of the monitoring location referenced to the specified Vertical - Datum. - altitude_accuracy : string or iterable of strings, optional - Accuracy of the altitude, in feet. An accuracy of +/- 0.1 foot would be - entered as “.1”. Many altitudes are interpolated from the contours on - topographic maps; accuracies determined in this way are generally - entered as one-half of the contour interval. - altitude_method_code : string or iterable of strings, optional - Codes representing the method used to measure altitude. - altitude_method_name : string or iterable of strings, optional - The name of the method used to measure altitude. - vertical_datum : string or iterable of strings, optional - The datum used to determine altitude and vertical position at the - monitoring location. - vertical_datum_name : string or iterable of strings, optional - The datum used to determine altitude and vertical position at the - monitoring location. - horizontal_positional_accuracy_code : string or iterable of strings, optional - Indicates the accuracy of the latitude longitude values. - horizontal_positional_accuracy : string or iterable of strings, optional - Indicates the accuracy of the latitude longitude values. - horizontal_position_method_code : string or iterable of strings, optional - Indicates the method used to determine latitude longitude values. - horizontal_position_method_name : string or iterable of strings, optional - Indicates the method used to determine latitude longitude values. - original_horizontal_datum : string or iterable of strings, optional - Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System - 1984. This field indicates the original datum used to determine - coordinates before they were converted. - original_horizontal_datum_name : string or iterable of strings, optional - Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System - 1984. This field indicates the original datum used to determine coordinates - before they were converted. - drainage_area : string or iterable of strings, optional - The area enclosed by a topographic divide from which direct surface runoff - from precipitation normally drains by gravity into the stream above that - point. - contributing_drainage_area : string or iterable of strings, optional - The contributing drainage area of a lake, stream, wetland, or estuary - monitoring location, in square miles. This item should be present only - if the contributing area is different from the total drainage area. This - situation can occur when part of the drainage area consists of very - porous soil or depressions that either allow all runoff to enter the - groundwater or trap the water in ponds so that rainfall does not - contribute to runoff. A transbasin diversion can also affect the total - drainage area. - time_zone_abbreviation : string or iterable of strings, optional - A short code describing the time zone used by a monitoring location. - uses_daylight_savings : string or iterable of strings, optional - A flag indicating whether or not a monitoring location uses daylight savings. - construction_date : string or iterable of strings, optional - Date the well was completed. - aquifer_code : string or iterable of strings, optional - Local aquifers in the USGS water resources data base are identified by a - geohydrologic unit code (a three-digit number related to the age of the - formation, followed by a 4 or 5 character abbreviation for the geologic - unit or aquifer name). Additional information is available - `at this link `_. - national_aquifer_code : string or iterable of strings, optional - National aquifers are the principal aquifers or aquifer systems in the United - States, defined as regionally extensive aquifers or aquifer systems that have - the potential to be used as a source of potable water. Not all groundwater - monitoring locations can be associated with a National Aquifer. Such - monitoring locations will not be retrieved using this search criteria. A `list - of National aquifer codes and names `_ - is available. - aquifer_type_code : string or iterable of strings, optional - Groundwater occurs in aquifers under two different conditions. Where water - only partly fills an aquifer, the upper surface is free to rise and decline. - These aquifers are referred to as unconfined (or water-table) aquifers. Where - water completely fills an aquifer that is overlain by a confining bed, the - aquifer is referred to as a confined (or artesian) aquifer. When a confined - aquifer is penetrated by a well, the water level in the well will rise above - the top of the aquifer (but not necessarily above land surface). Additional - information is available `at this link `_. - well_constructed_depth : string or iterable of strings, optional - The depth of the finished well, in feet below land surface datum. Note: Not - all groundwater monitoring locations have information on Well Depth. Such - monitoring locations will not be retrieved using this search criteria. - hole_constructed_depth : string or iterable of strings, optional - The total depth to which the hole is drilled, in feet below land surface datum. - Note: Not all groundwater monitoring locations have information on Hole Depth. - Such monitoring locations will not be retrieved using this search criteria. - depth_source_code : string or iterable of strings, optional - A code indicating the source of water-level data. A `list of - codes `_ - is available. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, id, agency_code, agency_name, - monitoring_location_number, monitoring_location_name, district_code, - country_code, country_name, state_code, state_name, county_code, - county_name, minor_civil_division_code, site_type_code, site_type, - hydrologic_unit_code, basin_code, altitude, altitude_accuracy, - altitude_method_code, altitude_method_name, vertical_datum, - vertical_datum_name, horizontal_positional_accuracy_code, - horizontal_positional_accuracy, horizontal_position_method_code, - horizontal_position_method_name, original_horizontal_datum, - original_horizontal_datum_name, drainage_area, - contributing_drainage_area, time_zone_abbreviation, - uses_daylight_savings, construction_date, aquifer_code, - national_aquifer_code, aquifer_type_code, well_constructed_depth, - hole_constructed_depth, depth_source_code. - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get monitoring locations within a bounding box - >>> # and leave out geometry - >>> df, md = dataretrieval.waterdata.get_monitoring_locations( - ... bbox=[-90.2, 42.6, -88.7, 43.2], skip_geometry=True - ... ) - - >>> # Get monitoring location info for specific sites - >>> # and only specific properties - >>> df, md = dataretrieval.waterdata.get_monitoring_locations( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], - ... properties=["monitoring_location_id", "state_name", "country_name"], - ... ) - """ - service = "monitoring-locations" - - # Build argument dictionary, omitting None values (resolving the unified - # `state` argument into the OGC `state_name` queryable). - args = _get_args( - _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} - ) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_time_series_metadata( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - parameter_name: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - hydrologic_unit_code: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_name: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - begin_utc: str | Iterable[str] | None = None, - end_utc: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - computation_period_identifier: str | Iterable[str] | None = None, - computation_identifier: str | Iterable[str] | None = None, - thresholds: float | list[float] | None = None, - sublocation_identifier: str | Iterable[str] | None = None, - primary: str | Iterable[str] | None = None, - parent_time_series_id: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - web_description: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data and continuous measurements are grouped into time series, - which represent a collection of observations of a single parameter, - potentially aggregated using a standard statistic, at a single monitoring - location. This endpoint provides metadata about those time series, - including their operational thresholds, units of measurement, and when - the earliest and most recent observations in a time series occurred. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter - codes and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - parameter_name : string or iterable of strings, optional - A human-understandable name corresponding to parameter_code. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. - Available options are: begin, begin_utc, computation_identifier, - computation_period_identifier, end, end_utc, geometry, - hydrologic_unit_code, id, last_modified, monitoring_location_id, - parameter_code, parameter_description, parameter_name, - parent_time_series_id, primary, state_name, statistic_id, - sublocation_identifier, thresholds, unit_of_measure, web_description - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - hydrologic_unit_code : string or iterable of strings, optional - The United States is divided and sub-divided into successively smaller - hydrologic units which are classified into four levels: regions, - sub-regions, accounting units, and cataloging units. The hydrologic - units are arranged within each other, from the smallest (cataloging units) - to the largest (regions). Each hydrologic unit is identified by a unique - hydrologic unit code (HUC) consisting of two to eight digits based on the - four levels of classification in the hydrologic unit system. - state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"``). - state_name : string or iterable of strings, optional - The name of the state or state equivalent in which the monitoring location - is located. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or "PT36H" - for the last 36 hours - - begin : string or iterable of strings, optional - This field contains the same information as "begin_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - end : string or iterable of strings, optional - This field contains the same information as "end_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - begin_utc : string or iterable of strings, optional - The datetime of the earliest observation in the time series. Together - with end, this field represents the period of record of a time series. - Note that some time series may have large gaps in their collection - record. This field is currently in the local time of the monitoring - location. We intend to update this in version v0 to use UTC with a time - zone. You can query this field using date-times or intervals, adhering - to RFC 3339, or using ISO 8601 duration objects. Intervals may be - bounded or half-bounded (double-dots at start or end). Only features - that have a begin that intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - end_utc : string or iterable of strings, optional - The datetime of the most recent observation in the time series. Data returned by - this endpoint updates at most once per day, and potentially less frequently than - that, and as such there may be more recent observations within a time series - than the time series end value reflects. Together with begin, this field - represents the period of record of a time series. It is additionally used to - determine whether a time series is "active". We intend to update this in - version v0 to use UTC with a time zone. - You can query this field using date-times or intervals, - adhering to RFC 3339, or using ISO 8601 duration objects. Intervals - may be bounded or half-bounded (double-dots at start or end). Only - features that have an end that intersects the value of datetime are - selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - computation_period_identifier : string or iterable of strings, optional - Indicates the period of data used for any statistical computations. - computation_identifier : string or iterable of strings, optional - Indicates whether the data from this time series represent a specific - statistical computation. - thresholds : number or list of numbers, optional - Thresholds represent known numeric limits for a time series, for example - the historic maximum value for a parameter or a level below which a - sensor is non-operative. These thresholds are sometimes used to - automatically determine if an observation is erroneous due to sensor - error, and therefore shouldn't be included in the time series. - sublocation_identifier : string or iterable of strings, optional - primary : string or iterable of strings, optional - parent_time_series_id : string or iterable of strings, optional - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - web_description : string or iterable of strings, optional - A description of what this time series represents, as used by WDFN and - other USGS data dissemination products. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get timeseries metadata information from a single site - >>> # over a yearlong period - >>> df, md = dataretrieval.waterdata.get_time_series_metadata( - ... monitoring_location_id="USGS-02238500" - ... ) - - >>> # Get timeseries metadata information from multiple sites - >>> # that begin after January 1, 1990. - >>> df, md = dataretrieval.waterdata.get_time_series_metadata( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], - ... begin="1990-01-01/..", - ... ) - """ - service = "time-series-metadata" - - # Build argument dictionary, omitting None values (resolving the unified - # `state` argument into the OGC `state_name` queryable). - args = _get_args( - _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} - ) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_combined_metadata( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - parameter_name: str | Iterable[str] | None = None, - parameter_description: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - data_type: str | Iterable[str] | None = None, - computation_identifier: str | Iterable[str] | None = None, - thresholds: float | list[float] | None = None, - sublocation_identifier: str | Iterable[str] | None = None, - primary: str | Iterable[str] | None = None, - parent_time_series_id: str | Iterable[str] | None = None, - web_description: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - agency_code: str | Iterable[str] | None = None, - agency_name: str | Iterable[str] | None = None, - monitoring_location_number: str | Iterable[str] | None = None, - monitoring_location_name: str | Iterable[str] | None = None, - district_code: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - country_name: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - state_name: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - county_name: str | Iterable[str] | None = None, - minor_civil_division_code: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type: str | Iterable[str] | None = None, - hydrologic_unit_code: str | Iterable[str] | None = None, - basin_code: str | Iterable[str] | None = None, - altitude: str | Iterable[str] | None = None, - altitude_accuracy: str | Iterable[str] | None = None, - altitude_method_code: str | Iterable[str] | None = None, - altitude_method_name: str | Iterable[str] | None = None, - vertical_datum: str | Iterable[str] | None = None, - vertical_datum_name: str | Iterable[str] | None = None, - horizontal_positional_accuracy_code: str | Iterable[str] | None = None, - horizontal_positional_accuracy: str | Iterable[str] | None = None, - horizontal_position_method_code: str | Iterable[str] | None = None, - horizontal_position_method_name: str | Iterable[str] | None = None, - original_horizontal_datum: str | Iterable[str] | None = None, - original_horizontal_datum_name: str | Iterable[str] | None = None, - drainage_area: str | Iterable[str] | None = None, - contributing_drainage_area: str | Iterable[str] | None = None, - time_zone_abbreviation: str | Iterable[str] | None = None, - uses_daylight_savings: str | Iterable[str] | None = None, - construction_date: str | Iterable[str] | None = None, - aquifer_code: str | Iterable[str] | None = None, - national_aquifer_code: str | Iterable[str] | None = None, - aquifer_type_code: str | Iterable[str] | None = None, - well_constructed_depth: str | Iterable[str] | None = None, - hole_constructed_depth: str | Iterable[str] | None = None, - depth_source_code: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get combined monitoring-location and time-series metadata. - - The ``combined-metadata`` collection joins the monitoring-locations - catalog with the time-series-metadata catalog so that one row is - returned per (location, parameter, statistic) inventory entry, - carrying every column from both source endpoints. This makes it the - most flexible "what data is available" endpoint in the Water Data - API: any monitoring-location attribute (state, HUC, site type, - drainage area, well-construction depth, …) can be combined with any - time-series attribute (parameter code, statistic, data type, period - of record, …) in a single query. - - See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/combined-metadata - - All ~35 location-catalog kwargs are accepted (``agency_code``, - ``state_name``, ``drainage_area``, ``aquifer_code``, …) but only - the most-used ones are documented below; see - :func:`get_monitoring_locations` for per-field descriptions. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. - Created by combining the agency code (e.g. ``USGS``) with the ID - number (e.g. ``02238500``), separated by a hyphen - (e.g. ``"USGS-02238500"``). - parameter_code : string or iterable of strings, optional - 5-digit codes used to identify the constituent measured and the - units of measure. See - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - parameter_name : string or iterable of strings, optional - A human-understandable name corresponding to ``parameter_code``. - parameter_description : string or iterable of strings, optional - A human-readable description of what is being measured. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement - associated with an observation. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents - (e.g. ``00001`` max, ``00002`` min, ``00003`` mean). Full list at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - data_type : string or iterable of strings, optional - The type of data the time series represents, e.g. - ``"Continuous values"``, ``"Daily values"``, - ``"Field measurements"``. - computation_identifier : string or iterable of strings, optional - Indicates whether the data from this time series represent a - specific statistical computation. - thresholds : number or list of numbers, optional - Numeric limits known for a time series (e.g. historic maximum, - below-which-the-sensor-is-non-operative). - sublocation_identifier : string or iterable of strings, optional - primary : string or iterable of strings, optional - A flag identifying whether the time series is "primary". Primary - time series are standard observations that have undergone Bureau - review and approval. Non-primary (provisional) time series have a - missing ``primary`` value, are produced for timely best-science - use, and are retained by this system for only 120 days. - parent_time_series_id : string or iterable of strings, optional - web_description : string or iterable of strings, optional - A description of what this time series represents, as used by - WDFN and other USGS data dissemination products. - last_modified, begin, end : string, optional - Datetime fields that accept either an RFC 3339 datetime, an - interval (``"start/end"``, optionally half-bounded with ``..``), - or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See - :func:`get_time_series_metadata` for the full grammar. - state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full - name (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a - two-digit ANSI/FIPS code (``"55"``). - state_name, county_name, hydrologic_unit_code, site_type, \ -site_type_code : string or iterable of strings, optional - Common location-catalog filters carried over from the - ``monitoring-locations`` collection. The function also accepts - the full list of location-catalog kwargs (agency, district, - altitude, vertical/horizontal datum, drainage area, aquifer, - well construction, …); see :func:`get_monitoring_locations` for - descriptions of each. - properties : string or iterable of strings, optional - Subset of columns to return. Defaults to every available - property. - skip_geometry : boolean, optional - Skip per-feature geometries; the returned object will be a plain - ``DataFrame`` with no spatial information. The Water Data APIs - use camelCase ``skipGeometry`` in CQL2 queries. - bbox : list of numbers, optional - Only features whose geometry intersects the bounding box are - selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 - (longitude/latitude, west-south-east-north). - limit : int, optional - Page size; the maximum allowable value is 50000. Default - (``None``) requests the maximum allowable limit. This is a - per-page size, not a cap on the total result: a query matching more - rows than ``limit`` still returns every matching row across - multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object pertaining to the query. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # All time series and field measurements at a single surface-water site - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... monitoring_location_id="USGS-05407000" - ... ) - - >>> # Same, for a groundwater well — water-level and aquifer columns - >>> # are populated where the surface-water example has nulls - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... monitoring_location_id="USGS-375907091432201" - ... ) - - >>> # Every series in a single county, useful for area-of-interest workflows - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... state="Wisconsin", county_name="Dane County" - ... ) - - >>> # Inventory across multiple HUCs, restricted to streams and springs - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... hydrologic_unit_code=["11010008", "11010009"], - ... site_type=["Stream", "Spring"], - ... ) - - >>> # Discharge time series at three sites with at least one - >>> # observation in the past month - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... monitoring_location_id=[ - ... "USGS-07069000", - ... "USGS-07064000", - ... "USGS-07068000", - ... ], - ... end="P1M", - ... parameter_code="00060", - ... ) - - >>> # Two-step "what's available?" → "fetch it" workflow: - >>> # 1. inventory the sites in two HUCs - >>> hucs, _ = dataretrieval.waterdata.get_combined_metadata( - ... hydrologic_unit_code=["11010008", "11010009"], - ... site_type="Stream", - ... ) - >>> # 2. pull continuous discharge at every distinct site found - >>> sites = hucs["monitoring_location_id"].unique().tolist() - >>> df, md = dataretrieval.waterdata.get_continuous( - ... monitoring_location_id=sites, - ... parameter_code="00060", - ... time="P1D", - ... ) - - """ - service = "combined-metadata" - - # Resolve the unified `state` argument into the OGC `state_name` queryable. - args = _get_args( - _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} - ) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_latest_continuous( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - latest_continuous_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """This endpoint provides the most recent observation for each time series - of continuous data. Continuous data are collected via automated sensors - installed at a monitoring location. They are collected at a high frequency - and often at a fixed 15-minute interval. Depending on the specific monitoring - location, the data may be transmitted automatically via telemetry and be - available on WDFN within minutes of collection, while other times the delivery - of data may be delayed if the monitoring location does not have the capacity to - automatically transmit data. Continuous data are described by parameter name - and parameter code. These data might also be referred to as "instantaneous - values" or "IV". - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, id, time_series_id, monitoring_location_id, - parameter_code, statistic_id, time, value, unit_of_measure, - approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - latest_continuous_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get latest flow data from a single site - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id="USGS-02238500", parameter_code="00060" - ... ) - - >>> # Restrict to the last 7 days; sites with no observation in that - >>> # window are dropped instead of returned with stale values - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... time="P7D", - ... ) - - >>> # Pull only rows whose underlying record was refreshed in the - >>> # last 7 days, across multiple sites and parameters - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id=["USGS-451605097071701", "USGS-14181500"], - ... parameter_code=["00060", "72019"], - ... last_modified="P7D", - ... ) - - >>> # Get latest continuous measurements for multiple sites - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] - ... ) - """ - service = "latest-continuous" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_latest_daily( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - latest_daily_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data provide one data value to represent water conditions for the - day. - - Throughout much of the history of the USGS, the primary water data available - was daily data collected manually at the monitoring location once each day. - With improved availability of computer storage and automated transmission of - data, the daily data published today are generally a statistical summary or - metric of the continuous data collected each day, such as the daily mean, - minimum, or maximum value. Daily data are automatically calculated from the - continuous data of the same parameter code and are described by parameter - code and a statistic code. These data have also been referred to as “daily - values” or “DV”. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, id, time_series_id, monitoring_location_id, - parameter_code, statistic_id, time, value, unit_of_measure, - approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - latest_daily_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get most recent daily flow data from a single site - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id="USGS-02238500", parameter_code="00060" - ... ) - - >>> # Restrict to rows whose underlying record was refreshed in the - >>> # last 7 days - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... last_modified="P7D", - ... ) - - >>> # Multi-site, multi-parameter — discharge and water temperature - >>> # at two sites in a single round-trip - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id=["USGS-01491000", "USGS-01645000"], - ... parameter_code=["00060", "00010"], - ... ) - - >>> # Get most recent daily measurements for two sites - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] - ... ) - """ - service = "latest-daily" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_field_measurements( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - observing_procedure_code: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - field_visit_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - observing_procedure: str | Iterable[str] | None = None, - vertical_datum: str | Iterable[str] | None = None, - measuring_agency: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Field measurements are physically measured values collected during a - visit to the monitoring location. Field measurements consist of measurements - of gage height and discharge, and readings of groundwater levels, and are - primarily used as calibration readings for the automated sensors collecting - continuous data. They are collected at a low frequency, and delivery of the - data in WDFN may be delayed due to data processing time. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - observing_procedure_code : string or iterable of strings, optional - A short code corresponding to the observing procedure for the field - measurement. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. See the - field-measurements schema in the OpenAPI reference for the available - columns (e.g. geometry, id, monitoring_location_id, parameter_code, - value, unit_of_measure, approval_status, qualifier, last_modified): - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements - field_visit_id : string or iterable of strings, optional - A universally unique identifier (UUID) for the field visit. - Multiple measurements may be made during a single field visit. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - observing_procedure : string or iterable of strings, optional - Water measurement or water-quality observing procedure descriptions. - vertical_datum : string or iterable of strings, optional - The datum used to determine altitude and vertical position at the - monitoring location. - measuring_agency : string or iterable of strings, optional - The agency performing the measurement. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using date-times - or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. - Intervals may be bounded or half-bounded (double-dots at start or end). - Only features that have a time that intersects the value of datetime are - selected. If a feature has multiple temporal properties, it is the - decision of the server whether only a single temporal property is used - to determine the extent or all relevant temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get field measurements from a single groundwater site - >>> # and parameter code, and do not return geometry - >>> df, md = dataretrieval.waterdata.get_field_measurements( - ... monitoring_location_id="USGS-375907091432201", - ... parameter_code="72019", - ... skip_geometry=True, - ... ) - - >>> # Half-bounded time range: every measurement at this site since - >>> # 1980 (open-ended end). Use ``"../"`` for the inverse - >>> # (everything up to a date). - >>> df, md = dataretrieval.waterdata.get_field_measurements( - ... monitoring_location_id="USGS-425957088141001", - ... time="1980-01-01/..", - ... ) - - >>> # Get field measurements from multiple sites and - >>> # parameter codes from the last 20 years - >>> df, md = dataretrieval.waterdata.get_field_measurements( - ... monitoring_location_id=[ - ... "USGS-451605097071701", - ... "USGS-263819081585801", - ... ], - ... parameter_code=["62611", "72019"], - ... time="P20Y", - ... ) - """ - service = "field-measurements" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_field_measurements_metadata( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - parameter_name: str | Iterable[str] | None = None, - parameter_description: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get field-measurement metadata: one row per (location, parameter) series. - - Each row describes a single field-measurement series — what parameter is - measured at the location, the period of record (``begin`` / ``end``), the - units, and so on — without returning the underlying observations - themselves. Use :func:`get_field_measurements` to fetch the values. - - This is the discrete-measurement analogue to - :func:`get_time_series_metadata` (which describes daily and continuous - series). It's primarily useful for inventory queries: "what - field-measurement parameters does this site have, and over what date - range?" - - See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements-metadata - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location, in - ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). - parameter_code : string or iterable of strings, optional - 5-digit parameter code. See - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - parameter_name : string or iterable of strings, optional - A human-understandable name corresponding to ``parameter_code``. - parameter_description : string or iterable of strings, optional - A human-readable description of what is being measured. - begin, end, last_modified : string, optional - Datetime fields that accept either an RFC 3339 datetime, an - interval (``"start/end"``, optionally half-bounded with ``..``), - or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See - :func:`get_time_series_metadata` for the full grammar. - properties : string or iterable of strings, optional - Subset of columns to return. Defaults to every available property. - skip_geometry : boolean, optional - Skip per-feature geometries; the returned object will be a plain - ``DataFrame`` with no spatial information. - bbox : list of numbers, optional - Only features whose geometry intersects the bounding box are - selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 - (longitude / latitude, west-south-east-north). - limit : int, optional - Page size; the maximum allowable value is 50000. Default - (``None``) requests the maximum allowable limit. This is a - per-page size, not a cap on the total result: a query matching more - rows than ``limit`` still returns every matching row across - multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object pertaining to the query. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # All field-measurement series at a surface-water site - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id="USGS-02238500" - ... ) - - >>> # Same, for a groundwater well - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id="USGS-375907091432201" - ... ) - - >>> # Multi-site, narrowed to two parameter codes - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id=[ - ... "USGS-451605097071701", - ... "USGS-263819081585801", - ... ], - ... parameter_code=["62611", "72019"], - ... ) - - >>> # Series modified in the last year — useful for incremental ETL - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id="USGS-375907091432201", - ... parameter_code="72019", - ... last_modified="P1Y", - ... ) - - """ - service = "field-measurements-metadata" - - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_peaks( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - water_year: int | list[int] | None = None, - year: int | list[int] | None = None, - month: int | list[int] | None = None, - day: int | list[int] | None = None, - peak_since: int | list[int] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get the annual peak streamflow / stage record for a monitoring location. - - Peaks are the largest values observed at a site each water year and are - the standard input to flood-frequency analysis (e.g. log-Pearson Type III - fits). The endpoint returns one row per (monitoring location, parameter, - water year), with the peak ``value`` and the ``time`` it occurred. - - The collection covers both stage (parameter ``"00065"``, ``ft``) and - discharge (parameter ``"00060"``, ``ft^3/s``); a typical streamgage has a - series for each. Reference docs: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/peaks - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location, in - ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). - parameter_code : string or iterable of strings, optional - 5-digit parameter code. Most peaks records are ``"00060"`` (discharge) - or ``"00065"`` (stage / gage height). Full list at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - time_series_id : string or iterable of strings, optional - ID of the time series the peak belongs to. - unit_of_measure : string or iterable of strings, optional - Human-readable units (e.g. ``"ft^3/s"``, ``"ft"``). - time : string, optional - Datetime, interval, or duration filter on the peak's date. - See :func:`get_time_series_metadata` for the full grammar. - last_modified : string, optional - Same datetime grammar as ``time``; filters on the database - last-modified timestamp (useful for incremental ETL polling). - water_year, year, month, day : int or list of ints, optional - Calendar / water-year filters on the peak event. The water year ends - September 30 (e.g. WY2024 = Oct 1, 2023 – Sep 30, 2024). - peak_since : int or list of ints, optional - Filter on the year since which the peak value has stood as the - record (the API serves this field as an integer; many rows are - ``null``). - properties : string or iterable of strings, optional - Subset of columns to return. Defaults to every available property. - skip_geometry : boolean, optional - Skip per-feature geometries; the returned object will be a plain - ``DataFrame`` with no spatial information. - bbox : list of numbers, optional - Only features whose geometry intersects the bounding box are - selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 - (longitude / latitude, west-south-east-north). - limit : int, optional - Page size; the maximum allowable value is 50000. Default - (``None``) requests the maximum allowable limit. This is a - per-page size, not a cap on the total result: a query matching more - rows than ``limit`` still returns every matching row across - multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object pertaining to the query. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Full annual peak record at one site (both stage and discharge) - >>> df, md = dataretrieval.waterdata.get_peaks( - ... monitoring_location_id="USGS-02238500" - ... ) - - >>> # Discharge peaks only - >>> df, md = dataretrieval.waterdata.get_peaks( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... ) - - >>> # Multi-site peaks for a parameter, narrowed to a water-year range - >>> df, md = dataretrieval.waterdata.get_peaks( - ... monitoring_location_id=[ - ... "USGS-07069000", - ... "USGS-07064000", - ... "USGS-07068000", - ... ], - ... parameter_code="00060", - ... water_year=[2020, 2021, 2022, 2023], - ... ) - - """ - service = "peaks" - - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_reference_table( - collection: str, - limit: int | None = None, - query: dict[str, Any] | None = None, - max_rows: int | None = None, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get metadata reference tables for the USGS Water Data API. - - Reference tables provide the range of allowable values for parameter - arguments in the waterdata module. - - Parameters - ---------- - collection : string - One of the following options: "agency-codes", "altitude-datums", - "aquifer-codes", "aquifer-types", "coordinate-accuracy-codes", - "coordinate-datum-codes", "coordinate-method-codes", "counties", - "hydrologic-unit-codes", "medium-codes", "national-aquifer-codes", - "parameter-codes", "reliability-codes", "site-types", "states", - "statistic-codes", "topographic-codes", "time-zone-codes" - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - query: dictionary, optional - The optional query parameter can be used to pass a dictionary of - query parameters to the collection API call. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole table. Useful for cheaply - previewing large tables (e.g. ``hydrologic-unit-codes`` has ~125k - rows). Unlike ``limit`` (the per-page size), this bounds the total - result. The default (None) downloads every page. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. The primary metadata - of each reference table will show up in the first column, where - the name of the column is the singular form of the collection name, - separated by underscores (e.g. the "medium-codes" reference table - has a column called "medium_code", which contains all possible - medium code values). - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object including the URL request and query time. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get table of USGS parameter codes - >>> ref, md = dataretrieval.waterdata.get_reference_table( - ... collection="parameter-codes" - ... ) - - >>> # Get table of selected USGS parameter codes - >>> ref, md = dataretrieval.waterdata.get_reference_table( - ... collection="parameter-codes", - ... query={"id": "00001,00002"}, - ... ) - """ - valid_code_services = get_args(METADATA_COLLECTIONS) - if collection not in valid_code_services: - raise ValueError( - f"Invalid code service: '{collection}'. " - f"Valid options are: {valid_code_services}." - ) - - # Give the ID column the collection name, singularized and underscored. - if collection == "counties": - output_id = "county" - elif collection.endswith("s"): - output_id = collection[:-1].replace("-", "_") - else: - output_id = collection.replace("-", "_") - - query_args = dict(query) if query else {} - if limit is not None: - query_args["limit"] = limit - return get_ogc_data( - args=query_args, output_id=output_id, service=collection, max_rows=max_rows - ) - - -def get_queryables(collection: str) -> tuple[pd.DataFrame, BaseMetadata]: - """List the queryable properties of a Water Data API collection. - - Every OGC collection (``daily``, ``continuous``, ``monitoring-locations``, - ...) advertises the set of properties that can be filtered on -- exposed as - the typed keyword arguments of the matching ``get_*`` function, and usable - directly in a CQL2 ``filter``. This returns that set, so the available - filters can be discovered programmatically and monitored for upstream - additions. - - Parameters - ---------- - collection : string - The collection id, e.g. ``"daily"``, ``"continuous"``, - ``"monitoring-locations"``, or ``"time-series-metadata"``. See - :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` for the data - collections; reference collections (e.g. ``"parameter-codes"``) work - too. - - Returns - ------- - df : ``pandas.DataFrame`` - One row per queryable, sorted by name, with columns ``queryable`` (the - property name), ``type``, ``title``, and ``description``. - md : :class:`dataretrieval.utils.BaseMetadata` - Metadata describing the request (URL, query time, response headers). - - Raises - ------ - DataRetrievalError - On an HTTP error response (e.g. an unknown ``collection`` yields a 404), - the typed subclass for the status. - - Examples - -------- - .. doctest:: - :skipif: True # network - - >>> from dataretrieval import waterdata - >>> df, md = waterdata.get_queryables("daily") - >>> df.set_index("queryable").loc["state_name", "type"] - 'string' - """ - # The OGC queryables document is a JSON Schema whose ``properties`` map each - # filterable property name to a ``{title, type, description}`` definition. - body, response = _check_ogc_requests(endpoint=collection, req_type="queryables") - properties: dict[str, Any] = body.get("properties", {}) - df = pd.DataFrame( - [ - { - "queryable": name, - "type": prop.get("type"), - "title": prop.get("title"), - "description": (prop.get("description") or "").strip(), - } - for name, prop in sorted(properties.items()) - ], - columns=["queryable", "type", "title", "description"], - ) - return df, BaseMetadata(response) - - -def get_codes(code_service: CODE_SERVICES) -> tuple[pd.DataFrame, BaseMetadata]: - """Return codes from a Samples code service. - - Parameters - ---------- - code_service : string - One of the following options: "states", "counties", "countries", - "sitetype", "samplemedia", "characteristicgroup", "characteristics", - or "observedproperty" - - Returns - ------- - df : ``pandas.DataFrame`` - The requested code table. - md : :obj:`dataretrieval.utils.BaseMetadata` - Metadata for the query (URL, query time, response headers). - """ - valid_code_services = get_args(CODE_SERVICES) - if code_service not in valid_code_services: - raise ValueError( - f"Invalid code service: '{code_service}'. " - f"Valid options are: {valid_code_services}." - ) - - url = f"{SAMPLES_URL}/codeservice/{code_service}?mimeType=application%2Fjson" - - response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) - - _raise_for_non_200(response) - - data_dict = json.loads(response.text) - data_list = data_dict["data"] - - df = pd.DataFrame(data_list) - - return df, BaseMetadata(response) - - -def _get_samples_csv( - url: str, params: dict[str, Any], ssl_check: bool -) -> tuple[pd.DataFrame, httpx.Response]: - """Issue a Samples CSV request and parse the body into a DataFrame. - - Shared tail for the Samples getters: sends the GET with the standard - headers (including ``X-Api-Key``), raises a typed error on a non-200 - (consistent with the OGC/stats path) instead of a bare - ``HTTPStatusError``, and reads the CSV. The caller wraps the response - as metadata and applies any per-getter post-step. - """ - logger.debug("Request: %s", httpx.URL(url).copy_merge_params(params)) - response = _get( - url, - params=params, - verify=ssl_check, - headers=_default_headers(url), - **HTTPX_DEFAULTS, - ) - _raise_for_non_200(response) - df = pd.read_csv(StringIO(response.text), delimiter=",") - return df, response - - -# Map the public snake_case ``get_samples`` parameters to the camelCase query -# parameter names the Samples API expects on the wire. ``characteristic`` is -# already snake_case-compatible (single word) and is sent unchanged. The -# remaining snake_case params are bookkeeping (``service``/``profile``/ -# ``ssl_check``) and never reach the request. -_SAMPLES_PARAM_TO_API = { - "activity_media_name": "activityMediaName", - "activity_start_date_lower": "activityStartDateLower", - "activity_start_date_upper": "activityStartDateUpper", - "activity_type_code": "activityTypeCode", - "characteristic_group": "characteristicGroup", - "characteristic_user_supplied": "characteristicUserSupplied", - "bbox": "boundingBox", - "country_code": "countryFips", - "state_code": "stateFips", - "county_code": "countyFips", - "site_type_code": "siteTypeCode", - "site_type_name": "siteTypeName", - "usgs_pcode": "usgsPCode", - "hydrologic_unit": "hydrologicUnit", - "monitoring_location_id": "monitoringLocationIdentifier", - "organization_id": "organizationIdentifier", - "point_location_latitude": "pointLocationLatitude", - "point_location_longitude": "pointLocationLongitude", - "point_location_within_miles": "pointLocationWithinMiles", - "project_id": "projectIdentifier", - "record_identifier_user_supplied": "recordIdentifierUserSupplied", -} - -# Deprecated camelCase keyword names (the Samples-API spelling) accepted for -# backward compatibility, mapped to the new snake_case parameter names. Derived -# from ``_SAMPLES_PARAM_TO_API`` so the two never drift apart. -_SAMPLES_LEGACY_KWARGS = { - api_name: py_name for py_name, api_name in _SAMPLES_PARAM_TO_API.items() -} - - -@_accept_legacy_kwargs(_SAMPLES_LEGACY_KWARGS) -def get_samples( - ssl_check: bool = True, - service: SERVICES = "results", - profile: PROFILES = "fullphyschem", - activity_media_name: str | Iterable[str] | None = None, - activity_start_date_lower: str | None = None, - activity_start_date_upper: str | None = None, - activity_type_code: str | Iterable[str] | None = None, - characteristic_group: str | Iterable[str] | None = None, - characteristic: str | Iterable[str] | None = None, - characteristic_user_supplied: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - country_code: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type_name: str | Iterable[str] | None = None, - usgs_pcode: str | Iterable[str] | None = None, - hydrologic_unit: str | Iterable[str] | None = None, - monitoring_location_id: str | Iterable[str] | None = None, - organization_id: str | Iterable[str] | None = None, - point_location_latitude: float | None = None, - point_location_longitude: float | None = None, - point_location_within_miles: float | None = None, - project_id: str | Iterable[str] | None = None, - record_identifier_user_supplied: str | Iterable[str] | None = None, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Search Samples database for USGS water quality data. - This is a wrapper function for the Samples database API. All potential - filters are provided as arguments to the function, but please do not - populate all possible filters; leave as many as feasible with their default - value (None). This is important because overcomplicated web service queries - can bog down the database's ability to return an applicable dataset before - it times out. - - The web GUI for the Samples database can be found here: - https://waterdata.usgs.gov/download-samples/#dataProfile=site - - If you would like more details on feasible query parameters (complete with - examples), please visit the Samples database swagger docs, here: - https://api.waterdata.usgs.gov/samples-data/docs#/ - - Parameters - ---------- - ssl_check : bool, optional - Check the SSL certificate. - service : string - One of the available Samples services: "results", "locations", "activities", - "projects", or "organizations". Defaults to "results". - profile : string - One of the available profiles associated with a service. Options for each - service are: - results - "fullphyschem", "basicphyschem", - "fullbio", "basicbio", "narrow", - "resultdetectionquantitationlimit", - "labsampleprep", "count" - locations - "site", "count" - activities - "sampact", "actmetric", - "actgroup", "count" - projects - "project", "projectmonitoringlocationweight" - organizations - "organization", "count" - activity_media_name : string or iterable of strings, optional - Name or code indicating environmental medium in which sample was taken. - Call ``get_codes("samplemedia")`` for the valid inputs. - Example: "Water". (Samples API: ``activityMediaName``) - activity_start_date_lower : string, optional - The start date if using a date range. Takes the format YYYY-MM-DD. - The logic is inclusive, i.e. it will also return results that - match the date. If left as None, will pull all data on or before - ``activity_start_date_upper``, if populated. - (Samples API: ``activityStartDateLower``) - activity_start_date_upper : string, optional - The end date if using a date range. Takes the format YYYY-MM-DD. - The logic is inclusive, i.e. it will also return results that - match the date. If left as None, will pull all data after - ``activity_start_date_lower`` up to the most recent available results. - (Samples API: ``activityStartDateUpper``) - activity_type_code : string or iterable of strings, optional - Text code that describes type of field activity performed. - Example: "Sample-Routine, regular". (Samples API: ``activityTypeCode``) - characteristic_group : string or iterable of strings, optional - Characteristic group is a broad category of characteristics - describing one or more results. Call ``get_codes("characteristicgroup")`` - for the valid inputs. - Example: "Organics, PFAS" (Samples API: ``characteristicGroup``) - characteristic : string or iterable of strings, optional - Characteristic is a specific category describing one or more results. - Call ``get_codes("characteristics")`` for the valid inputs. - Example: "Suspended Sediment Discharge" (Samples API: ``characteristic``) - characteristic_user_supplied : string or iterable of strings, optional - A user supplied characteristic name describing one or more results. - (Samples API: ``characteristicUserSupplied``) - bbox : list of four floats, optional - Filters on the associated monitoring location's point location - by checking if it is located within the specified geographic area. - The logic is inclusive, i.e. it will include locations that overlap - with the edge of the bounding box. Values are separated by commas, - expressed in decimal degrees, NAD83, and longitudes west of Greenwich - are negative. The format is a list consisting of: - - * Western-most longitude - * Southern-most latitude - * Eastern-most longitude - * Northern-most latitude - - Example: [-92.8,44.2,-88.9,46.0] (Samples API: ``boundingBox``) - country_code : string or iterable of strings, optional - Example: "US" (United States) (Samples API: ``countryFips``) - state_code : string or iterable of strings, optional - Call ``get_codes("states")`` for the valid inputs. - Example: "US:15" (United States: Hawaii) (Samples API: ``stateFips``) - county_code : string or iterable of strings, optional - Call ``get_codes("counties")`` for the valid inputs. - Example: "US:15:001" (United States: Hawaii, Hawaii County) - (Samples API: ``countyFips``) - site_type_code : string or iterable of strings, optional - An abbreviation for a certain site type. Call ``get_codes("sitetype")`` - for the valid inputs. - Example: "GW" (Groundwater site) (Samples API: ``siteTypeCode``) - site_type_name : string or iterable of strings, optional - A full name for a certain site type. Call ``get_codes("sitetype")`` - for the valid inputs. - Example: "Well" (Samples API: ``siteTypeName``) - usgs_pcode : string or iterable of strings, optional - 5-digit number used in the US Geological Survey computerized - data system, National Water Information System (NWIS), to - uniquely identify a specific constituent (the ``parameterCode`` column - of ``get_codes("characteristics")``). - Example: "00060" (Discharge, cubic feet per second) - (Samples API: ``usgsPCode``) - hydrologic_unit : string or iterable of strings, optional - Max 12-digit number used to describe a hydrologic unit. - Example: "070900020502" (Samples API: ``hydrologicUnit``) - monitoring_location_id : string or iterable of strings, optional - A monitoring location identifier has two parts: the agency code - and the location number, separated by a dash (-). - Example: "USGS-040851385" - (Samples API: ``monitoringLocationIdentifier``) - organization_id : string or iterable of strings, optional - Designator used to uniquely identify a specific organization. - Currently only accepting the organization "USGS". - (Samples API: ``organizationIdentifier``) - point_location_latitude : float, optional - Latitude for a point/radius query (decimal degrees). Must be used - with ``point_location_longitude`` and ``point_location_within_miles``. - (Samples API: ``pointLocationLatitude``) - point_location_longitude : float, optional - Longitude for a point/radius query (decimal degrees). Must be used - with ``point_location_latitude`` and ``point_location_within_miles``. - (Samples API: ``pointLocationLongitude``) - point_location_within_miles : float, optional - Radius for a point/radius query. Must be used with - ``point_location_latitude`` and ``point_location_longitude``. - (Samples API: ``pointLocationWithinMiles``) - project_id : string or iterable of strings, optional - Designator used to uniquely identify a data collection project. Project - identifiers are specific to an organization (e.g. USGS). - Example: "ZH003QW03" (Samples API: ``projectIdentifier``) - record_identifier_user_supplied : string or iterable of strings, optional - Internal AQS record identifier that returns 1 entry. Only available - for the "results" service. - (Samples API: ``recordIdentifierUserSupplied``) - - Returns - ------- - df : ``pandas.DataFrame`` - Formatted data returned from the API query. For each - ``Date`` / ``Time`` / ``TimeZone`` triplet in - the response (e.g. ``Activity_StartDate``, ``Activity_StartTime``, - ``Activity_StartTimeZone``), an additional ``DateTime`` column - is appended holding a UTC ``Timestamp`` derived from the three. The - original Date/Time/TimeZone columns are left intact; rows whose - timezone abbreviation is not recognized resolve to ``NaT``. Rows are - sorted by ``Activity_StartDateTime`` when present (the API's default - order is unstable). - md : :obj:`dataretrieval.utils.BaseMetadata` - Custom ``dataretrieval`` metadata object pertaining to the query. - - Examples - -------- - .. code:: - - >>> # Get PFAS results within a bounding box - >>> df, md = dataretrieval.waterdata.get_samples( - ... bbox=[-90.2, 42.6, -88.7, 43.2], - ... characteristic_group="Organics, PFAS", - ... ) - - >>> # Get all activities for the Commonwealth of Virginia over a date range - >>> df, md = dataretrieval.waterdata.get_samples( - ... service="activities", - ... profile="sampact", - ... activity_start_date_lower="2023-10-01", - ... activity_start_date_upper="2024-01-01", - ... state_code="US:51", - ... ) - - >>> # Get all pH samples for two sites in Utah - >>> df, md = dataretrieval.waterdata.get_samples( - ... monitoring_location_id=[ - ... "USGS-393147111462301", - ... "USGS-393343111454101", - ... ], - ... usgs_pcode="00400", - ... ) - - """ - - _check_profiles(service, profile) - - # Build argument dictionary, omitting None values. Parameters are the - # public snake_case names here; translate them to the camelCase names the - # Samples API expects just before building the request. - args = _get_args(locals(), exclude={"ssl_check", "profile"}) - params = {_SAMPLES_PARAM_TO_API.get(key, key): value for key, value in args.items()} - - params.update({"mimeType": "text/csv"}) - - if "boundingBox" in params: - params["boundingBox"] = to_str(params["boundingBox"]) - - url = f"{SAMPLES_URL}/{service}/{profile}" - - df, response = _get_samples_csv(url, params, ssl_check) - df = _attach_datetime_columns(df) - - return df, BaseMetadata(response) - - -@_accept_legacy_kwargs({"monitoringLocationIdentifier": "monitoring_location_id"}) -def get_samples_summary( - monitoring_location_id: str, - ssl_check: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get a summary of discrete water-quality samples at a single monitoring location. - - Wraps the Samples database summary service described at - https://api.waterdata.usgs.gov/samples-data/docs. The service returns one - row per (characteristic group, characteristic, user-supplied characteristic) - combination with result and activity counts and the first / most recent - activity dates — useful for taking inventory of what discrete-sample data - exists at a site before pulling the underlying observations with - :func:`get_samples`. - - The summary service is single-site only: it accepts exactly one monitoring - location per request. - - Parameters - ---------- - monitoring_location_id : string - A monitoring location identifier has two parts, separated by a dash - (``-``): the agency code and the location number. Examples: - ``"USGS-040851385"``, ``"AZ014-320821110580701"``, - ``"CAX01-15304600"``. Bare location numbers without an agency prefix - are accepted by the service but return an empty result, so a prefix - is effectively required. (Samples API: ``monitoringLocationIdentifier``) - ssl_check : bool, optional - Check the SSL certificate. Default is True. - - Returns - ------- - df : ``pandas.DataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - Custom ``dataretrieval`` metadata object pertaining to the query. - - Examples - -------- - .. code:: - - >>> # What discrete-sample data is available at this site? - >>> df, md = dataretrieval.waterdata.get_samples_summary( - ... monitoring_location_id="USGS-04074950" - ... ) - - """ - if not isinstance(monitoring_location_id, str): - raise TypeError( - "monitoring_location_id must be a string; the Samples " - "summary service accepts exactly one monitoring location per " - f"request, got {type(monitoring_location_id).__name__}." - ) - - url = f"{SAMPLES_URL}/summary/{quote(monitoring_location_id, safe='')}" - params = {"mimeType": "text/csv"} - - df, response = _get_samples_csv(url, params, ssl_check) - - return df, BaseMetadata(response) - - -def get_stats_por( - approval_status: str | None = None, - computation_type: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - start_date: str | None = None, - end_date: str | None = None, - monitoring_location_id: str | Iterable[str] | None = None, - page_size: int = 1000, - parent_time_series_id: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type_name: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - normal_type: str | None = None, - expand_percentiles: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get day-of-year and month-of-year water data statistics from the - USGS Water Data API. - This service (called the "observationNormals" endpoint on api.waterdata.usgs.gov) - provides endpoints for access to computations on the historical record regarding - water conditions, including minimum, maximum, mean, median, and percentiles for - day of year and month of year. For more information regarding the calculation of - statistics and other details, please visit the Statistics documentation page: - https://waterdata.usgs.gov/statistics-documentation/. - - Note: This API is under active beta development and subject to - change. Improved handling of significant figures will be - addressed in a future release. - - Parameters - ---------- - approval_status: string, optional - Whether to include approved and/or provisional observations. - At this time, only approved observations are returned. - computation_type: string, optional - Desired statistical computation method. Available values are: - arithmetic_mean, maximum, median, minimum, percentile. - country_code: string, optional - Country query parameter. API defaults to "US". - state: string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit - ANSI/FIPS code ("55"). - state_code: string, optional - State query parameter. Takes the format "US:XX", where XX is - the two-digit state code. API defaults to "US:42" (Pennsylvania). - county_code: string, optional - County query parameter. Takes the format "US:XX:YYY", where XX is - the two-digit state code and YYY is the three-digit county code. - API defaults to "US:42:103" (Pennsylvania, Pike County). - start_date: string or datetime, optional - Start day for the query in the month-day format (MM-DD). - end_date: string or datetime, optional - End day for the query in the month-day format (MM-DD). - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - page_size : int, optional - The number of results to return per page, where one result represents a - monitoring location. The default is 1000. - parent_time_series_id: string, optional - The parent_time_series_id returns statistics tied to a - particular database entry. - site_type_code: string, optional - Site type code query parameter. - A list of valid site type codes is available at: - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. - Example: "GW" (Groundwater site) - site_type_name: string, optional - Site type name query parameter. - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - normal_type : string, optional - Filter the returned normals to a single period. If unspecified - (default), all matching data are returned. Available values: - "DOY" (day-of-year) and "MOY" (month-of-year). - expand_percentiles : boolean - Percentile data for a given day of year or month of year by default - are returned from the service as lists of string values and percentile - thresholds in the "values" and "percentiles" columns, respectively. - When `expand_percentiles` is set to True (default), each value and - percentile threshold specific to a computation id are returned as - individual rows in the dataframe, with the value reported in the - "value" column and the corresponding percentile reported in a - "percentile" column (and the "values" and "percentiles" columns - are removed). Missing percentile values expressed as 'nan' in the - list of string values are removed from the dataframe to save space. - Setting `expand_percentiles` to False retains the "values" and - "percentiles" columns produced by the service. Including - both 'percentiles' and one or more other statistics ('median', - 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` - argument will return both the "values" column, containing the list - of percentile threshold values, and a "value" column, containing - the singular summary value for the other statistics. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object. - - Examples - -------- - .. code:: - - >>> # Get daily, monthly, and annual percentiles for streamflow at - >>> # a monitoring location of interest - >>> df, md = dataretrieval.waterdata.get_stats_por( - ... monitoring_location_id="USGS-05114000", - ... parameter_code="00060", - ... computation_type="percentile", - ... ) - - >>> # Get all daily and monthly statistics for the month of January - >>> # over the entire period of record for streamflow and gage height - >>> # at a monitoring location of interest - >>> df, md = dataretrieval.waterdata.get_stats_por( - ... monitoring_location_id="USGS-05114000", - ... parameter_code=["00060", "00065"], - ... start_date="01-01", - ... end_date="01-31", - ... ) - """ - # Build argument dictionary, omitting None values - params = _get_args( - _with_state(locals(), to="fips_us", into="state_code"), - exclude={"expand_percentiles"}, - ) - - return stats.get_data( - args=params, service="observationNormals", expand_percentiles=expand_percentiles - ) - - -def get_stats_date_range( - approval_status: str | None = None, - computation_type: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - start_date: str | None = None, - end_date: str | None = None, - monitoring_location_id: str | Iterable[str] | None = None, - page_size: int = 1000, - parent_time_series_id: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type_name: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - interval_type: str | Iterable[str] | None = None, - expand_percentiles: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get monthly and annual water data statistics from the USGS Water Data API. - This service (called the "observationIntervals" endpoint on api.waterdata.usgs.gov) - provides endpoints for access to computations on the historical record regarding - water conditions, including minimum, maximum, mean, median, and percentiles for - month-year, and water/calendar years. For more information regarding the calculation - of statistics and other details, please visit the Statistics documentation page: - https://waterdata.usgs.gov/statistics-documentation/. - - Note: This API is under active beta development and subject to - change. Improved handling of significant figures will be - addressed in a future release. - - Parameters - ---------- - approval_status: string, optional - Whether to include approved and/or provisional observations. - At this time, only approved observations are returned. - computation_type: string, optional - Desired statistical computation method. Available values are: - arithmetic_mean, maximum, median, minimum, percentile. - country_code: string, optional - Country query parameter. API defaults to "US". - state: string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit - ANSI/FIPS code ("55"). - state_code: string, optional - State query parameter. Takes the format "US:XX", where XX is - the two-digit state code. API defaults to "US:42" (Pennsylvania). - county_code: string, optional - County query parameter. Takes the format "US:XX:YYY", where XX is - the two-digit state code and YYY is the three-digit county code. - API defaults to "US:42:103" (Pennsylvania, Pike County). - start_date: string or datetime, optional - Start date for the query in the year-month-day format - (YYYY-MM-DD). - end_date: string or datetime, optional - End date for the query in the year-month-day format - (YYYY-MM-DD). - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - page_size : int, optional - The number of results to return per page, where one result represents a - monitoring location. The default is 1000. - parent_time_series_id: string, optional - The parent_time_series_id returns statistics tied to a - particular database entry. - site_type_code: string, optional - Site type code query parameter. - You can see a list of valid site type codes here: - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. - Example: "GW" (Groundwater site) - site_type_name: string, optional - Site type name query parameter. - You can see a list of valid site type names here: - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. - Example: "Well" - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - interval_type : string or iterable of strings, optional - Filter the returned intervals to one or more periods. If unspecified - (default), all matching data are returned. Available values: - "M" (month), "CY" (calendar year), and "WY" (water year). - expand_percentiles : boolean - Percentile data for a given day of year or month of year by default - are returned from the service as lists of string values and percentile - thresholds in the "values" and "percentiles" columns, respectively. - When `expand_percentiles` is set to True (default), each value and - percentile threshold specific to a computation id are returned as - individual rows in the dataframe, with the value reported in the - "value" column and the corresponding percentile reported in a - "percentile" column (and the "values" and "percentiles" columns - are removed). Missing percentile values expressed as 'nan' in the - list of string values are removed from the dataframe to save space. - Setting `expand_percentiles` to False retains the "values" and - "percentiles" columns produced by the service. Including - both 'percentiles' and one or more other statistics ('median', - 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` - argument will return both the "values" column, containing the list - of percentile threshold values, and a "value" column, containing - the singular summary value for the other statistics. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object. - - Examples - -------- - .. code:: - - >>> # Get monthly and yearly medians for streamflow at streams in Rhode Island - >>> # from calendar year 2024. - >>> df, md = dataretrieval.waterdata.get_stats_date_range( - ... state="RI", # Rhode Island (postal code, name, or FIPS all work) - ... parameter_code="00060", - ... site_type_code="ST", - ... start_date="2024-01-01", - ... end_date="2024-12-31", - ... computation_type="median", - ... ) - - >>> # Get monthly and yearly minimum and maximums for gage height at - >>> # a monitoring location of interest - >>> df, md = dataretrieval.waterdata.get_stats_date_range( - ... monitoring_location_id="USGS-05114000", - ... parameter_code="00065", - ... computation_type=["minimum", "maximum"], - ... ) - """ - # Build argument dictionary, omitting None values - params = _get_args( - _with_state(locals(), to="fips_us", into="state_code"), - exclude={"expand_percentiles"}, - ) - - return stats.get_data( - args=params, - service="observationIntervals", - expand_percentiles=expand_percentiles, - ) - - -def get_channel( - monitoring_location_id: str | Iterable[str] | None = None, - field_visit_id: str | Iterable[str] | None = None, - measurement_number: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, - channel_name: str | Iterable[str] | None = None, - channel_flow: str | Iterable[str] | None = None, - channel_flow_unit: str | Iterable[str] | None = None, - channel_width: str | Iterable[str] | None = None, - channel_width_unit: str | Iterable[str] | None = None, - channel_area: str | Iterable[str] | None = None, - channel_area_unit: str | Iterable[str] | None = None, - channel_velocity: str | Iterable[str] | None = None, - channel_velocity_unit: str | Iterable[str] | None = None, - channel_location_distance: str | Iterable[str] | None = None, - channel_location_distance_unit: str | Iterable[str] | None = None, - channel_stability: str | Iterable[str] | None = None, - channel_material: str | Iterable[str] | None = None, - channel_evenness: str | Iterable[str] | None = None, - horizontal_velocity_description: str | Iterable[str] | None = None, - vertical_velocity_description: str | Iterable[str] | None = None, - longitudinal_velocity_description: str | Iterable[str] | None = None, - measurement_type: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - channel_measurement_type: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """ - Channel measurements taken as part of streamflow field measurements. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - field_visit_id : string or iterable of strings, optional - A universally unique identifier (UUID) for the field visit. - Multiple measurements - may be made during a single field visit. - measurement_number : string or iterable of strings, optional - Measurement number. - time : string or iterable of strings, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or "PT36H" - for the last 36 hours - - channel_name : string or iterable of strings, optional - The channel name. - channel_flow : string or iterable of strings, optional - The channel discharge (flow). - channel_flow_unit : string or iterable of strings, optional - The units for channel discharge. - channel_width : string or iterable of strings, optional - The channel width. - channel_width_unit : string or iterable of strings, optional - The units for channel width. - channel_area : string or iterable of strings, optional - The channel area. - channel_area_unit : string or iterable of strings, optional - The units for channel area. - channel_velocity : string or iterable of strings, optional - The mean channel velocity. - channel_velocity_unit : string or iterable of strings, optional - The units for channel velocity. - channel_location_distance : string or iterable of strings, optional - The channel location distance. - channel_location_distance_unit : string or iterable of strings, optional - The units for channel location distance. - channel_stability : string or iterable of strings, optional - The stability of the channel material. - channel_material : string or iterable of strings, optional - The channel material. - channel_evenness : string or iterable of strings, optional - The channel evenness from bank to bank. - horizontal_velocity_description : string or iterable of strings, optional - The horizontal velocity description. - vertical_velocity_description : string or iterable of strings, optional - The vertical velocity description. - longitudinal_velocity_description : string or iterable of strings, optional - The longitudinal velocity description. - measurement_type : string or iterable of strings, optional - The type of channel measurement. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - channel_measurement_type : string or iterable of strings, optional - The channel measurement type. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, channel_measurements_id, monitoring_location_id, - field_visit_id, measurement_number, time, channel_name, channel_flow, - channel_flow_unit, channel_width, channel_width_unit, channel_area, - channel_area_unit, channel_velocity, channel_velocity_unit, - channel_location_distance, channel_location_distance_unit, channel_stability, - channel_material, channel_evenness, horizontal_velocity_description, - vertical_velocity_description, longitudinal_velocity_description, - measurement_type, last_modified, channel_measurement_type. The default - (None) will return all columns of the data. - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get channel data from a - >>> # single site from a single year - >>> df, md = dataretrieval.waterdata.get_channel( - ... monitoring_location_id="USGS-02238500", - ... ) - """ - service = "channel-measurements" - - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_cql( - service: WATERDATA_SERVICES, - cql: str | dict[str, Any], - *, - properties: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - skip_geometry: bool | None = None, - convert_type: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Query a Water Data OGC API collection with an arbitrary CQL2 filter. - - Sends ``cql`` as a CQL2 filter against ``service`` and returns the matching - features, shaped like the typed getters (``get_daily``, ``get_continuous``, - …): the wire ``id`` renamed to the service's id column, columns ordered and - sorted, and dtypes coerced. Use it when you need a predicate the typed - getters can't express — a top-level ``or``, ``like`` with ``%`` wildcards, - comparison operators, nested boolean trees, or a geometry predicate beyond a - bounding box; prefer a typed getter when one covers the query. - - The request is a single POST with the ``cql`` body sent verbatim, so there - are no multi-value arguments to chunk: narrow a query whose URL or body - would exceed the server's size cap rather than relying on automatic - chunking. - - The CQL2 grammar is documented at - https://api.waterdata.usgs.gov/docs/ogcapi/complex-queries/. - - Parameters - ---------- - service : str - OGC collection name. Must be one of - :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` - (e.g. ``"daily"``, ``"monitoring-locations"``). - cql : str or dict - CQL2 query. A ``dict`` is JSON-serialized for transport; a ``str`` is - sent through unchanged. The query goes into the HTTP POST body with - ``Content-Type: application/query-cql-json``. - properties : str or iterable of str, optional - Server-side property whitelist (passed as ``properties=`` on the URL). - Reduces payload size. ``"id"`` resolves to the service's ``output_id`` - (e.g. ``daily_id``) the same way it does in the typed wrappers. - bbox : list of float, optional - Bounding box ``[xmin, ymin, xmax, ymax]`` in CRS 4326. Combines with the - CQL filter as an additional spatial predicate. - limit : int, optional - Page size, clamped server-side to 50,000. - skip_geometry : bool, optional - If True, the server omits geometry from each feature - (``skipGeometry=true``). - convert_type : bool, default True - Coerce date/datetime/numeric columns to typed dtypes after the - DataFrame is built. - - Returns - ------- - df : pandas.DataFrame or geopandas.GeoDataFrame - Result of the query. GeoDataFrame when ``geopandas`` is installed and - geometry is present. - md : :class:`dataretrieval.utils.BaseMetadata` - Request metadata (URL, query time, response headers). - - Examples - -------- - .. code:: - - >>> # Daily values for two parameter codes at two sites - >>> # (compound AND-of-INs). - >>> from dataretrieval import waterdata - >>> cql = { - ... "op": "and", - ... "args": [ - ... { - ... "op": "in", - ... "args": [ - ... {"property": "parameter_code"}, - ... ["00060", "00065"], - ... ], - ... }, - ... { - ... "op": "in", - ... "args": [ - ... {"property": "monitoring_location_id"}, - ... ["USGS-07367300", "USGS-03277200"], - ... ], - ... }, - ... ], - ... } - >>> df, md = waterdata.get_cql(service="daily", cql=cql) - - >>> # Monitoring locations whose HUC starts with "02070010" - >>> # (LIKE with the CQL2 ``%`` wildcard). - >>> df, md = waterdata.get_cql( - ... service="monitoring-locations", - ... cql='{"op": "like", "args": [' - ... '{"property": "hydrologic_unit_code"},' - ... ' "02070010%"]}', - ... ) - """ - if service not in _OUTPUT_ID_BY_SERVICE: - raise ValueError( - f"Unknown service {service!r}. Valid services: " - f"{sorted(_OUTPUT_ID_BY_SERVICE)}." - ) - output_id = _OUTPUT_ID_BY_SERVICE[service] - - # ``dict`` is the pythonic input — serialize on the way out. ``str`` is sent - # verbatim so callers who already have a CQL2 doc (e.g. imported from a - # config file) don't need to re-parse it. - body = json.dumps(cql, separators=(",", ":")) if isinstance(cql, dict) else cql - - properties_list = _as_str_list(properties, "properties") - - # Drop id aliases (``daily_id``/``id``) and ``geometry`` from the wire - # ``properties`` (the feature ``id`` is always returned and renamed - # downstream), matching the typed getters. - wire_properties = _switch_properties_id(properties_list, output_id, service) - - req = _construct_cql_request( - service, - body, - properties=wire_properties, - bbox=bbox, - limit=limit, - skip_geometry=skip_geometry, - ) - - df, response = fetch_ogc_request(req, service=service) - - return _finalize_ogc( - df, - response, - properties=properties_list, - output_id=output_id, - convert_type=convert_type, - service=service, - ) +from dataretrieval.waterdata.utils import get_ogc_data as _get_ogc_data + +__all__ = [ + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_peaks", + "get_queryables", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", +] + +# Preserve the documented legacy implementation path for introspection and +# Sphinx while the function objects live in cohesive family modules. +for _name in __all__: + globals()[_name].__module__ = __name__ +del _name + +# Private compatibility names used by existing callers and patch targets. +_SAMPLES_PARAM_TO_API = _samples._SAMPLES_PARAM_TO_API +_SAMPLES_LEGACY_KWARGS = _samples._SAMPLES_LEGACY_KWARGS +get_ogc_data = _get_ogc_data diff --git a/dataretrieval/waterdata/cql.py b/dataretrieval/waterdata/cql.py new file mode 100644 index 00000000..1134ca86 --- /dev/null +++ b/dataretrieval/waterdata/cql.py @@ -0,0 +1,166 @@ +"""Generalized CQL2 request adapter for Water Data collections.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc import fetch_ogc_request +from dataretrieval.ogc.requests import ( + _as_str_list, + _construct_cql_request, + _switch_properties_id, +) +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.types import ( + WATERDATA_SERVICES, +) +from dataretrieval.waterdata.utils import ( + _OUTPUT_ID_BY_SERVICE, + _finalize_ogc, +) + + +def get_cql( + service: WATERDATA_SERVICES, + cql: str | dict[str, Any], + *, + properties: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + skip_geometry: bool | None = None, + convert_type: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Query a Water Data OGC API collection with an arbitrary CQL2 filter. + + Sends ``cql`` as a CQL2 filter against ``service`` and returns the matching + features, shaped like the typed getters (``get_daily``, ``get_continuous``, + …): the wire ``id`` renamed to the service's id column, columns ordered and + sorted, and dtypes coerced. Use it when you need a predicate the typed + getters can't express — a top-level ``or``, ``like`` with ``%`` wildcards, + comparison operators, nested boolean trees, or a geometry predicate beyond a + bounding box; prefer a typed getter when one covers the query. + + The request is a single POST with the ``cql`` body sent verbatim, so there + are no multi-value arguments to chunk: narrow a query whose URL or body + would exceed the server's size cap rather than relying on automatic + chunking. + + The CQL2 grammar is documented at + https://api.waterdata.usgs.gov/docs/ogcapi/complex-queries/. + + Parameters + ---------- + service : str + OGC collection name. Must be one of + :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` + (e.g. ``"daily"``, ``"monitoring-locations"``). + cql : str or dict + CQL2 query. A ``dict`` is JSON-serialized for transport; a ``str`` is + sent through unchanged. The query goes into the HTTP POST body with + ``Content-Type: application/query-cql-json``. + properties : str or iterable of str, optional + Server-side property whitelist (passed as ``properties=`` on the URL). + Reduces payload size. ``"id"`` resolves to the service's ``output_id`` + (e.g. ``daily_id``) the same way it does in the typed wrappers. + bbox : list of float, optional + Bounding box ``[xmin, ymin, xmax, ymax]`` in CRS 4326. Combines with the + CQL filter as an additional spatial predicate. + limit : int, optional + Page size, clamped server-side to 50,000. + skip_geometry : bool, optional + If True, the server omits geometry from each feature + (``skipGeometry=true``). + convert_type : bool, default True + Coerce date/datetime/numeric columns to typed dtypes after the + DataFrame is built. + + Returns + ------- + df : pandas.DataFrame or geopandas.GeoDataFrame + Result of the query. GeoDataFrame when ``geopandas`` is installed and + geometry is present. + md : :class:`dataretrieval.utils.BaseMetadata` + Request metadata (URL, query time, response headers). + + Examples + -------- + .. code:: + + >>> # Daily values for two parameter codes at two sites + >>> # (compound AND-of-INs). + >>> from dataretrieval import waterdata + >>> cql = { + ... "op": "and", + ... "args": [ + ... { + ... "op": "in", + ... "args": [ + ... {"property": "parameter_code"}, + ... ["00060", "00065"], + ... ], + ... }, + ... { + ... "op": "in", + ... "args": [ + ... {"property": "monitoring_location_id"}, + ... ["USGS-07367300", "USGS-03277200"], + ... ], + ... }, + ... ], + ... } + >>> df, md = waterdata.get_cql(service="daily", cql=cql) + + >>> # Monitoring locations whose HUC starts with "02070010" + >>> # (LIKE with the CQL2 ``%`` wildcard). + >>> df, md = waterdata.get_cql( + ... service="monitoring-locations", + ... cql='{"op": "like", "args": [' + ... '{"property": "hydrologic_unit_code"},' + ... ' "02070010%"]}', + ... ) + """ + if service not in _OUTPUT_ID_BY_SERVICE: + raise ValueError( + f"Unknown service {service!r}. Valid services: " + f"{sorted(_OUTPUT_ID_BY_SERVICE)}." + ) + output_id = _OUTPUT_ID_BY_SERVICE[service] + + # ``dict`` is the pythonic input — serialize on the way out. ``str`` is sent + # verbatim so callers who already have a CQL2 doc (e.g. imported from a + # config file) don't need to re-parse it. + body = json.dumps(cql, separators=(",", ":")) if isinstance(cql, dict) else cql + + properties_list = _as_str_list(properties, "properties") + + # Drop id aliases (``daily_id``/``id``) and ``geometry`` from the wire + # ``properties`` (the feature ``id`` is always returned and renamed + # downstream), matching the typed getters. + wire_properties = _switch_properties_id(properties_list, output_id, service) + + req = _construct_cql_request( + service, + body, + properties=wire_properties, + bbox=bbox, + limit=limit, + skip_geometry=skip_geometry, + ) + + df, response = fetch_ogc_request(req, service=service) + + return _finalize_ogc( + df, + response, + properties=properties_list, + output_id=output_id, + convert_type=convert_type, + service=service, + ) + + +__all__ = ["get_cql"] diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py new file mode 100644 index 00000000..ac8a1d54 --- /dev/null +++ b/dataretrieval/waterdata/measurements.py @@ -0,0 +1,576 @@ +"""Discrete field, peak, and channel measurement getters.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc.filters import FILTER_LANG +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.utils import ( + _get_args, + get_ogc_data, +) + + +def get_field_measurements( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + observing_procedure_code: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + field_visit_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + observing_procedure: str | Iterable[str] | None = None, + vertical_datum: str | Iterable[str] | None = None, + measuring_agency: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Field measurements are physically measured values collected during a + visit to the monitoring location. Field measurements consist of measurements + of gage height and discharge, and readings of groundwater levels, and are + primarily used as calibration readings for the automated sensors collecting + continuous data. They are collected at a low frequency, and delivery of the + data in WDFN may be delayed due to data processing time. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + observing_procedure_code : string or iterable of strings, optional + A short code corresponding to the observing procedure for the field + measurement. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. See the + field-measurements schema in the OpenAPI reference for the available + columns (e.g. geometry, id, monitoring_location_id, parameter_code, + value, unit_of_measure, approval_status, qualifier, last_modified): + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements + field_visit_id : string or iterable of strings, optional + A universally unique identifier (UUID) for the field visit. + Multiple measurements may be made during a single field visit. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + observing_procedure : string or iterable of strings, optional + Water measurement or water-quality observing procedure descriptions. + vertical_datum : string or iterable of strings, optional + The datum used to determine altitude and vertical position at the + monitoring location. + measuring_agency : string or iterable of strings, optional + The agency performing the measurement. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using date-times + or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. + Intervals may be bounded or half-bounded (double-dots at start or end). + Only features that have a time that intersects the value of datetime are + selected. If a feature has multiple temporal properties, it is the + decision of the server whether only a single temporal property is used + to determine the extent or all relevant temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get field measurements from a single groundwater site + >>> # and parameter code, and do not return geometry + >>> df, md = dataretrieval.waterdata.get_field_measurements( + ... monitoring_location_id="USGS-375907091432201", + ... parameter_code="72019", + ... skip_geometry=True, + ... ) + + >>> # Half-bounded time range: every measurement at this site since + >>> # 1980 (open-ended end). Use ``"../"`` for the inverse + >>> # (everything up to a date). + >>> df, md = dataretrieval.waterdata.get_field_measurements( + ... monitoring_location_id="USGS-425957088141001", + ... time="1980-01-01/..", + ... ) + + >>> # Get field measurements from multiple sites and + >>> # parameter codes from the last 20 years + >>> df, md = dataretrieval.waterdata.get_field_measurements( + ... monitoring_location_id=[ + ... "USGS-451605097071701", + ... "USGS-263819081585801", + ... ], + ... parameter_code=["62611", "72019"], + ... time="P20Y", + ... ) + """ + service = "field-measurements" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_peaks( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + time: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + water_year: int | list[int] | None = None, + year: int | list[int] | None = None, + month: int | list[int] | None = None, + day: int | list[int] | None = None, + peak_since: int | list[int] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get the annual peak streamflow / stage record for a monitoring location. + + Peaks are the largest values observed at a site each water year and are + the standard input to flood-frequency analysis (e.g. log-Pearson Type III + fits). The endpoint returns one row per (monitoring location, parameter, + water year), with the peak ``value`` and the ``time`` it occurred. + + The collection covers both stage (parameter ``"00065"``, ``ft``) and + discharge (parameter ``"00060"``, ``ft^3/s``); a typical streamgage has a + series for each. Reference docs: + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/peaks + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location, in + ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). + parameter_code : string or iterable of strings, optional + 5-digit parameter code. Most peaks records are ``"00060"`` (discharge) + or ``"00065"`` (stage / gage height). Full list at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + time_series_id : string or iterable of strings, optional + ID of the time series the peak belongs to. + unit_of_measure : string or iterable of strings, optional + Human-readable units (e.g. ``"ft^3/s"``, ``"ft"``). + time : string, optional + Datetime, interval, or duration filter on the peak's date. + See :func:`get_time_series_metadata` for the full grammar. + last_modified : string, optional + Same datetime grammar as ``time``; filters on the database + last-modified timestamp (useful for incremental ETL polling). + water_year, year, month, day : int or list of ints, optional + Calendar / water-year filters on the peak event. The water year ends + September 30 (e.g. WY2024 = Oct 1, 2023 – Sep 30, 2024). + peak_since : int or list of ints, optional + Filter on the year since which the peak value has stood as the + record (the API serves this field as an integer; many rows are + ``null``). + properties : string or iterable of strings, optional + Subset of columns to return. Defaults to every available property. + skip_geometry : boolean, optional + Skip per-feature geometries; the returned object will be a plain + ``DataFrame`` with no spatial information. + bbox : list of numbers, optional + Only features whose geometry intersects the bounding box are + selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 + (longitude / latitude, west-south-east-north). + limit : int, optional + Page size; the maximum allowable value is 50000. Default + (``None``) requests the maximum allowable limit. This is a + per-page size, not a cap on the total result: a query matching more + rows than ``limit`` still returns every matching row across + multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object pertaining to the query. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Full annual peak record at one site (both stage and discharge) + >>> df, md = dataretrieval.waterdata.get_peaks( + ... monitoring_location_id="USGS-02238500" + ... ) + + >>> # Discharge peaks only + >>> df, md = dataretrieval.waterdata.get_peaks( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... ) + + >>> # Multi-site peaks for a parameter, narrowed to a water-year range + >>> df, md = dataretrieval.waterdata.get_peaks( + ... monitoring_location_id=[ + ... "USGS-07069000", + ... "USGS-07064000", + ... "USGS-07068000", + ... ], + ... parameter_code="00060", + ... water_year=[2020, 2021, 2022, 2023], + ... ) + + """ + service = "peaks" + + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_channel( + monitoring_location_id: str | Iterable[str] | None = None, + field_visit_id: str | Iterable[str] | None = None, + measurement_number: str | Iterable[str] | None = None, + time: str | Iterable[str] | None = None, + channel_name: str | Iterable[str] | None = None, + channel_flow: str | Iterable[str] | None = None, + channel_flow_unit: str | Iterable[str] | None = None, + channel_width: str | Iterable[str] | None = None, + channel_width_unit: str | Iterable[str] | None = None, + channel_area: str | Iterable[str] | None = None, + channel_area_unit: str | Iterable[str] | None = None, + channel_velocity: str | Iterable[str] | None = None, + channel_velocity_unit: str | Iterable[str] | None = None, + channel_location_distance: str | Iterable[str] | None = None, + channel_location_distance_unit: str | Iterable[str] | None = None, + channel_stability: str | Iterable[str] | None = None, + channel_material: str | Iterable[str] | None = None, + channel_evenness: str | Iterable[str] | None = None, + horizontal_velocity_description: str | Iterable[str] | None = None, + vertical_velocity_description: str | Iterable[str] | None = None, + longitudinal_velocity_description: str | Iterable[str] | None = None, + measurement_type: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + channel_measurement_type: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """ + Channel measurements taken as part of streamflow field measurements. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + field_visit_id : string or iterable of strings, optional + A universally unique identifier (UUID) for the field visit. + Multiple measurements + may be made during a single field visit. + measurement_number : string or iterable of strings, optional + Measurement number. + time : string or iterable of strings, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or "PT36H" + for the last 36 hours + + channel_name : string or iterable of strings, optional + The channel name. + channel_flow : string or iterable of strings, optional + The channel discharge (flow). + channel_flow_unit : string or iterable of strings, optional + The units for channel discharge. + channel_width : string or iterable of strings, optional + The channel width. + channel_width_unit : string or iterable of strings, optional + The units for channel width. + channel_area : string or iterable of strings, optional + The channel area. + channel_area_unit : string or iterable of strings, optional + The units for channel area. + channel_velocity : string or iterable of strings, optional + The mean channel velocity. + channel_velocity_unit : string or iterable of strings, optional + The units for channel velocity. + channel_location_distance : string or iterable of strings, optional + The channel location distance. + channel_location_distance_unit : string or iterable of strings, optional + The units for channel location distance. + channel_stability : string or iterable of strings, optional + The stability of the channel material. + channel_material : string or iterable of strings, optional + The channel material. + channel_evenness : string or iterable of strings, optional + The channel evenness from bank to bank. + horizontal_velocity_description : string or iterable of strings, optional + The horizontal velocity description. + vertical_velocity_description : string or iterable of strings, optional + The vertical velocity description. + longitudinal_velocity_description : string or iterable of strings, optional + The longitudinal velocity description. + measurement_type : string or iterable of strings, optional + The type of channel measurement. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + channel_measurement_type : string or iterable of strings, optional + The channel measurement type. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, channel_measurements_id, monitoring_location_id, + field_visit_id, measurement_number, time, channel_name, channel_flow, + channel_flow_unit, channel_width, channel_width_unit, channel_area, + channel_area_unit, channel_velocity, channel_velocity_unit, + channel_location_distance, channel_location_distance_unit, channel_stability, + channel_material, channel_evenness, horizontal_velocity_description, + vertical_velocity_description, longitudinal_velocity_description, + measurement_type, last_modified, channel_measurement_type. The default + (None) will return all columns of the data. + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get channel data from a + >>> # single site from a single year + >>> df, md = dataretrieval.waterdata.get_channel( + ... monitoring_location_id="USGS-02238500", + ... ) + """ + service = "channel-measurements" + + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +__all__ = ["get_field_measurements", "get_peaks", "get_channel"] diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py new file mode 100644 index 00000000..4c288f1a --- /dev/null +++ b/dataretrieval/waterdata/metadata.py @@ -0,0 +1,994 @@ +"""Monitoring-location and data-inventory metadata getters.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc.filters import FILTER_LANG +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.utils import ( + _get_args, + _with_state, + get_ogc_data, +) + + +def get_monitoring_locations( + monitoring_location_id: str | Iterable[str] | None = None, + agency_code: str | Iterable[str] | None = None, + agency_name: str | Iterable[str] | None = None, + monitoring_location_number: str | Iterable[str] | None = None, + monitoring_location_name: str | Iterable[str] | None = None, + district_code: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + country_name: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + state_name: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + county_name: str | Iterable[str] | None = None, + minor_civil_division_code: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type: str | Iterable[str] | None = None, + hydrologic_unit_code: str | Iterable[str] | None = None, + basin_code: str | Iterable[str] | None = None, + altitude: str | Iterable[str] | None = None, + altitude_accuracy: str | Iterable[str] | None = None, + altitude_method_code: str | Iterable[str] | None = None, + altitude_method_name: str | Iterable[str] | None = None, + vertical_datum: str | Iterable[str] | None = None, + vertical_datum_name: str | Iterable[str] | None = None, + horizontal_positional_accuracy_code: str | Iterable[str] | None = None, + horizontal_positional_accuracy: str | Iterable[str] | None = None, + horizontal_position_method_code: str | Iterable[str] | None = None, + horizontal_position_method_name: str | Iterable[str] | None = None, + original_horizontal_datum: str | Iterable[str] | None = None, + original_horizontal_datum_name: str | Iterable[str] | None = None, + drainage_area: str | Iterable[str] | None = None, + contributing_drainage_area: str | Iterable[str] | None = None, + time_zone_abbreviation: str | Iterable[str] | None = None, + uses_daylight_savings: str | Iterable[str] | None = None, + construction_date: str | Iterable[str] | None = None, + aquifer_code: str | Iterable[str] | None = None, + national_aquifer_code: str | Iterable[str] | None = None, + aquifer_type_code: str | Iterable[str] | None = None, + well_constructed_depth: str | Iterable[str] | None = None, + hole_constructed_depth: str | Iterable[str] | None = None, + depth_source_code: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Location information is basic information about the monitoring location + including the name, identifier, agency responsible for data collection, and + the date the location was established. It also includes information about + the type of location, such as stream, lake, or groundwater, and geographic + information about the location, such as state, county, latitude and + longitude, and hydrologic unit code (HUC). + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + agency_code : string or iterable of strings, optional + The agency that is reporting the data. Agency codes are fixed values + assigned by the National Water Information System (NWIS). + agency_name : string or iterable of strings, optional + The name of the agency that is reporting the data. + monitoring_location_number : string or iterable of strings, optional + Each monitoring location in the USGS data base has a unique 8- to + 15-digit identification number. Monitoring location numbers are + assigned based on this logic: + https://help.waterdata.usgs.gov/faq/sites/do-station-numbers-have-any-particular-meaning. + monitoring_location_name : string or iterable of strings, optional + This is the official name of the monitoring location in the database. + For well information this can be a district-assigned local number. + district_code : string or iterable of strings, optional + The Water Science Centers (WSCs) across the United States use the FIPS + state code as the district code. In some cases, monitoring locations and + samples may be managed by a water science center that is adjacent to the + state in which the monitoring location actually resides. For example a + monitoring location may have a district code of 30 which translates to + Montana, but the state code could be 56 for Wyoming because that is where + the monitoring location actually is located. + country_code : string or iterable of strings, optional + The code for the country in which the monitoring location is located. + country_name : string or iterable of strings, optional + The name of the country in which the monitoring location is located. + state : string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit + ANSI/FIPS code (``"55"``). + state_code : string or iterable of strings, optional + State code. A two-digit ANSI code (formerly FIPS code) as defined by + the American National Standards Institute, to define States and + equivalents. A three-digit ANSI code is used to define counties and + county equivalents. A `lookup table + `_ + is available. The only countries with + political subdivisions other than the US are Mexico and Canada. The Mexican + states have US state codes ranging from 81-86 and Canadian provinces have + state codes ranging from 90-98. + state_name : string or iterable of strings, optional + The name of the state or state equivalent in which the monitoring location + is located. + county_code : string or iterable of strings, optional + The code for the county or county equivalent (parish, borough, etc.) in which + the monitoring location is located. A `list of codes + `__ is available. + county_name : string or iterable of strings, optional + The name of the county or county equivalent (parish, borough, etc.) in which + the monitoring location is located. A `list of codes + `__ is available. + minor_civil_division_code : string or iterable of strings, optional + Codes for primary governmental or administrative divisions of the county or + county equivalent in which the monitoring location is located. + site_type_code : string or iterable of strings, optional + A code describing the hydrologic setting of the monitoring location. + site_type : string or iterable of strings, optional + A description of the hydrologic setting of the monitoring location. + hydrologic_unit_code : string or iterable of strings, optional + The United States is divided and sub-divided into successively smaller + hydrologic units which are classified into four levels: regions, + sub-regions, accounting units, and cataloging units. The hydrologic + units are arranged within each other, from the smallest (cataloging + units) to the largest (regions). Each hydrologic unit is identified by a + unique hydrologic unit code (HUC) consisting of two to eight digits + based on the four levels of classification in the hydrologic unit + system. + basin_code : string or iterable of strings, optional + The Basin Code or "drainage basin code" is a two-digit code that further + subdivides the 8-digit hydrologic-unit code. The drainage basin code is + defined by the USGS State Office where the monitoring location is + located. + altitude : string or iterable of strings, optional + Altitude of the monitoring location referenced to the specified Vertical + Datum. + altitude_accuracy : string or iterable of strings, optional + Accuracy of the altitude, in feet. An accuracy of +/- 0.1 foot would be + entered as “.1”. Many altitudes are interpolated from the contours on + topographic maps; accuracies determined in this way are generally + entered as one-half of the contour interval. + altitude_method_code : string or iterable of strings, optional + Codes representing the method used to measure altitude. + altitude_method_name : string or iterable of strings, optional + The name of the method used to measure altitude. + vertical_datum : string or iterable of strings, optional + The datum used to determine altitude and vertical position at the + monitoring location. + vertical_datum_name : string or iterable of strings, optional + The datum used to determine altitude and vertical position at the + monitoring location. + horizontal_positional_accuracy_code : string or iterable of strings, optional + Indicates the accuracy of the latitude longitude values. + horizontal_positional_accuracy : string or iterable of strings, optional + Indicates the accuracy of the latitude longitude values. + horizontal_position_method_code : string or iterable of strings, optional + Indicates the method used to determine latitude longitude values. + horizontal_position_method_name : string or iterable of strings, optional + Indicates the method used to determine latitude longitude values. + original_horizontal_datum : string or iterable of strings, optional + Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System + 1984. This field indicates the original datum used to determine + coordinates before they were converted. + original_horizontal_datum_name : string or iterable of strings, optional + Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System + 1984. This field indicates the original datum used to determine coordinates + before they were converted. + drainage_area : string or iterable of strings, optional + The area enclosed by a topographic divide from which direct surface runoff + from precipitation normally drains by gravity into the stream above that + point. + contributing_drainage_area : string or iterable of strings, optional + The contributing drainage area of a lake, stream, wetland, or estuary + monitoring location, in square miles. This item should be present only + if the contributing area is different from the total drainage area. This + situation can occur when part of the drainage area consists of very + porous soil or depressions that either allow all runoff to enter the + groundwater or trap the water in ponds so that rainfall does not + contribute to runoff. A transbasin diversion can also affect the total + drainage area. + time_zone_abbreviation : string or iterable of strings, optional + A short code describing the time zone used by a monitoring location. + uses_daylight_savings : string or iterable of strings, optional + A flag indicating whether or not a monitoring location uses daylight savings. + construction_date : string or iterable of strings, optional + Date the well was completed. + aquifer_code : string or iterable of strings, optional + Local aquifers in the USGS water resources data base are identified by a + geohydrologic unit code (a three-digit number related to the age of the + formation, followed by a 4 or 5 character abbreviation for the geologic + unit or aquifer name). Additional information is available + `at this link `_. + national_aquifer_code : string or iterable of strings, optional + National aquifers are the principal aquifers or aquifer systems in the United + States, defined as regionally extensive aquifers or aquifer systems that have + the potential to be used as a source of potable water. Not all groundwater + monitoring locations can be associated with a National Aquifer. Such + monitoring locations will not be retrieved using this search criteria. A `list + of National aquifer codes and names `_ + is available. + aquifer_type_code : string or iterable of strings, optional + Groundwater occurs in aquifers under two different conditions. Where water + only partly fills an aquifer, the upper surface is free to rise and decline. + These aquifers are referred to as unconfined (or water-table) aquifers. Where + water completely fills an aquifer that is overlain by a confining bed, the + aquifer is referred to as a confined (or artesian) aquifer. When a confined + aquifer is penetrated by a well, the water level in the well will rise above + the top of the aquifer (but not necessarily above land surface). Additional + information is available `at this link `_. + well_constructed_depth : string or iterable of strings, optional + The depth of the finished well, in feet below land surface datum. Note: Not + all groundwater monitoring locations have information on Well Depth. Such + monitoring locations will not be retrieved using this search criteria. + hole_constructed_depth : string or iterable of strings, optional + The total depth to which the hole is drilled, in feet below land surface datum. + Note: Not all groundwater monitoring locations have information on Hole Depth. + Such monitoring locations will not be retrieved using this search criteria. + depth_source_code : string or iterable of strings, optional + A code indicating the source of water-level data. A `list of + codes `_ + is available. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, id, agency_code, agency_name, + monitoring_location_number, monitoring_location_name, district_code, + country_code, country_name, state_code, state_name, county_code, + county_name, minor_civil_division_code, site_type_code, site_type, + hydrologic_unit_code, basin_code, altitude, altitude_accuracy, + altitude_method_code, altitude_method_name, vertical_datum, + vertical_datum_name, horizontal_positional_accuracy_code, + horizontal_positional_accuracy, horizontal_position_method_code, + horizontal_position_method_name, original_horizontal_datum, + original_horizontal_datum_name, drainage_area, + contributing_drainage_area, time_zone_abbreviation, + uses_daylight_savings, construction_date, aquifer_code, + national_aquifer_code, aquifer_type_code, well_constructed_depth, + hole_constructed_depth, depth_source_code. + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get monitoring locations within a bounding box + >>> # and leave out geometry + >>> df, md = dataretrieval.waterdata.get_monitoring_locations( + ... bbox=[-90.2, 42.6, -88.7, 43.2], skip_geometry=True + ... ) + + >>> # Get monitoring location info for specific sites + >>> # and only specific properties + >>> df, md = dataretrieval.waterdata.get_monitoring_locations( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], + ... properties=["monitoring_location_id", "state_name", "country_name"], + ... ) + """ + service = "monitoring-locations" + + # Build argument dictionary, omitting None values (resolving the unified + # `state` argument into the OGC `state_name` queryable). + args = _get_args( + _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} + ) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_time_series_metadata( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + parameter_name: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + hydrologic_unit_code: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_name: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + begin: str | Iterable[str] | None = None, + end: str | Iterable[str] | None = None, + begin_utc: str | Iterable[str] | None = None, + end_utc: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + computation_period_identifier: str | Iterable[str] | None = None, + computation_identifier: str | Iterable[str] | None = None, + thresholds: float | list[float] | None = None, + sublocation_identifier: str | Iterable[str] | None = None, + primary: str | Iterable[str] | None = None, + parent_time_series_id: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + web_description: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Daily data and continuous measurements are grouped into time series, + which represent a collection of observations of a single parameter, + potentially aggregated using a standard statistic, at a single monitoring + location. This endpoint provides metadata about those time series, + including their operational thresholds, units of measurement, and when + the earliest and most recent observations in a time series occurred. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter + codes and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + parameter_name : string or iterable of strings, optional + A human-understandable name corresponding to parameter_code. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. + Available options are: begin, begin_utc, computation_identifier, + computation_period_identifier, end, end_utc, geometry, + hydrologic_unit_code, id, last_modified, monitoring_location_id, + parameter_code, parameter_description, parameter_name, + parent_time_series_id, primary, state_name, statistic_id, + sublocation_identifier, thresholds, unit_of_measure, web_description + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + hydrologic_unit_code : string or iterable of strings, optional + The United States is divided and sub-divided into successively smaller + hydrologic units which are classified into four levels: regions, + sub-regions, accounting units, and cataloging units. The hydrologic + units are arranged within each other, from the smallest (cataloging units) + to the largest (regions). Each hydrologic unit is identified by a unique + hydrologic unit code (HUC) consisting of two to eight digits based on the + four levels of classification in the hydrologic unit system. + state : string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit + ANSI/FIPS code (``"55"``). + state_name : string or iterable of strings, optional + The name of the state or state equivalent in which the monitoring location + is located. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or "PT36H" + for the last 36 hours + + begin : string or iterable of strings, optional + This field contains the same information as "begin_utc", but in the + local time of the monitoring location. It is retained for backwards + compatibility, but will be removed in V1 of these APIs. + end : string or iterable of strings, optional + This field contains the same information as "end_utc", but in the + local time of the monitoring location. It is retained for backwards + compatibility, but will be removed in V1 of these APIs. + begin_utc : string or iterable of strings, optional + The datetime of the earliest observation in the time series. Together + with end, this field represents the period of record of a time series. + Note that some time series may have large gaps in their collection + record. This field is currently in the local time of the monitoring + location. We intend to update this in version v0 to use UTC with a time + zone. You can query this field using date-times or intervals, adhering + to RFC 3339, or using ISO 8601 duration objects. Intervals may be + bounded or half-bounded (double-dots at start or end). Only features + that have a begin that intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + end_utc : string or iterable of strings, optional + The datetime of the most recent observation in the time series. Data returned by + this endpoint updates at most once per day, and potentially less frequently than + that, and as such there may be more recent observations within a time series + than the time series end value reflects. Together with begin, this field + represents the period of record of a time series. It is additionally used to + determine whether a time series is "active". We intend to update this in + version v0 to use UTC with a time zone. + You can query this field using date-times or intervals, + adhering to RFC 3339, or using ISO 8601 duration objects. Intervals + may be bounded or half-bounded (double-dots at start or end). Only + features that have an end that intersects the value of datetime are + selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + computation_period_identifier : string or iterable of strings, optional + Indicates the period of data used for any statistical computations. + computation_identifier : string or iterable of strings, optional + Indicates whether the data from this time series represent a specific + statistical computation. + thresholds : number or list of numbers, optional + Thresholds represent known numeric limits for a time series, for example + the historic maximum value for a parameter or a level below which a + sensor is non-operative. These thresholds are sometimes used to + automatically determine if an observation is erroneous due to sensor + error, and therefore shouldn't be included in the time series. + sublocation_identifier : string or iterable of strings, optional + primary : string or iterable of strings, optional + parent_time_series_id : string or iterable of strings, optional + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + web_description : string or iterable of strings, optional + A description of what this time series represents, as used by WDFN and + other USGS data dissemination products. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get timeseries metadata information from a single site + >>> # over a yearlong period + >>> df, md = dataretrieval.waterdata.get_time_series_metadata( + ... monitoring_location_id="USGS-02238500" + ... ) + + >>> # Get timeseries metadata information from multiple sites + >>> # that begin after January 1, 1990. + >>> df, md = dataretrieval.waterdata.get_time_series_metadata( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], + ... begin="1990-01-01/..", + ... ) + """ + service = "time-series-metadata" + + # Build argument dictionary, omitting None values (resolving the unified + # `state` argument into the OGC `state_name` queryable). + args = _get_args( + _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} + ) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_combined_metadata( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + parameter_name: str | Iterable[str] | None = None, + parameter_description: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + data_type: str | Iterable[str] | None = None, + computation_identifier: str | Iterable[str] | None = None, + thresholds: float | list[float] | None = None, + sublocation_identifier: str | Iterable[str] | None = None, + primary: str | Iterable[str] | None = None, + parent_time_series_id: str | Iterable[str] | None = None, + web_description: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + begin: str | Iterable[str] | None = None, + end: str | Iterable[str] | None = None, + agency_code: str | Iterable[str] | None = None, + agency_name: str | Iterable[str] | None = None, + monitoring_location_number: str | Iterable[str] | None = None, + monitoring_location_name: str | Iterable[str] | None = None, + district_code: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + country_name: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + state_name: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + county_name: str | Iterable[str] | None = None, + minor_civil_division_code: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type: str | Iterable[str] | None = None, + hydrologic_unit_code: str | Iterable[str] | None = None, + basin_code: str | Iterable[str] | None = None, + altitude: str | Iterable[str] | None = None, + altitude_accuracy: str | Iterable[str] | None = None, + altitude_method_code: str | Iterable[str] | None = None, + altitude_method_name: str | Iterable[str] | None = None, + vertical_datum: str | Iterable[str] | None = None, + vertical_datum_name: str | Iterable[str] | None = None, + horizontal_positional_accuracy_code: str | Iterable[str] | None = None, + horizontal_positional_accuracy: str | Iterable[str] | None = None, + horizontal_position_method_code: str | Iterable[str] | None = None, + horizontal_position_method_name: str | Iterable[str] | None = None, + original_horizontal_datum: str | Iterable[str] | None = None, + original_horizontal_datum_name: str | Iterable[str] | None = None, + drainage_area: str | Iterable[str] | None = None, + contributing_drainage_area: str | Iterable[str] | None = None, + time_zone_abbreviation: str | Iterable[str] | None = None, + uses_daylight_savings: str | Iterable[str] | None = None, + construction_date: str | Iterable[str] | None = None, + aquifer_code: str | Iterable[str] | None = None, + national_aquifer_code: str | Iterable[str] | None = None, + aquifer_type_code: str | Iterable[str] | None = None, + well_constructed_depth: str | Iterable[str] | None = None, + hole_constructed_depth: str | Iterable[str] | None = None, + depth_source_code: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get combined monitoring-location and time-series metadata. + + The ``combined-metadata`` collection joins the monitoring-locations + catalog with the time-series-metadata catalog so that one row is + returned per (location, parameter, statistic) inventory entry, + carrying every column from both source endpoints. This makes it the + most flexible "what data is available" endpoint in the Water Data + API: any monitoring-location attribute (state, HUC, site type, + drainage area, well-construction depth, …) can be combined with any + time-series attribute (parameter code, statistic, data type, period + of record, …) in a single query. + + See the OpenAPI reference for the full list of supported fields: + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/combined-metadata + + All ~35 location-catalog kwargs are accepted (``agency_code``, + ``state_name``, ``drainage_area``, ``aquifer_code``, …) but only + the most-used ones are documented below; see + :func:`get_monitoring_locations` for per-field descriptions. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. + Created by combining the agency code (e.g. ``USGS``) with the ID + number (e.g. ``02238500``), separated by a hyphen + (e.g. ``"USGS-02238500"``). + parameter_code : string or iterable of strings, optional + 5-digit codes used to identify the constituent measured and the + units of measure. See + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + parameter_name : string or iterable of strings, optional + A human-understandable name corresponding to ``parameter_code``. + parameter_description : string or iterable of strings, optional + A human-readable description of what is being measured. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement + associated with an observation. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents + (e.g. ``00001`` max, ``00002`` min, ``00003`` mean). Full list at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + data_type : string or iterable of strings, optional + The type of data the time series represents, e.g. + ``"Continuous values"``, ``"Daily values"``, + ``"Field measurements"``. + computation_identifier : string or iterable of strings, optional + Indicates whether the data from this time series represent a + specific statistical computation. + thresholds : number or list of numbers, optional + Numeric limits known for a time series (e.g. historic maximum, + below-which-the-sensor-is-non-operative). + sublocation_identifier : string or iterable of strings, optional + primary : string or iterable of strings, optional + A flag identifying whether the time series is "primary". Primary + time series are standard observations that have undergone Bureau + review and approval. Non-primary (provisional) time series have a + missing ``primary`` value, are produced for timely best-science + use, and are retained by this system for only 120 days. + parent_time_series_id : string or iterable of strings, optional + web_description : string or iterable of strings, optional + A description of what this time series represents, as used by + WDFN and other USGS data dissemination products. + last_modified, begin, end : string, optional + Datetime fields that accept either an RFC 3339 datetime, an + interval (``"start/end"``, optionally half-bounded with ``..``), + or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See + :func:`get_time_series_metadata` for the full grammar. + state : string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full + name (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a + two-digit ANSI/FIPS code (``"55"``). + state_name, county_name, hydrologic_unit_code, site_type, \ +site_type_code : string or iterable of strings, optional + Common location-catalog filters carried over from the + ``monitoring-locations`` collection. The function also accepts + the full list of location-catalog kwargs (agency, district, + altitude, vertical/horizontal datum, drainage area, aquifer, + well construction, …); see :func:`get_monitoring_locations` for + descriptions of each. + properties : string or iterable of strings, optional + Subset of columns to return. Defaults to every available + property. + skip_geometry : boolean, optional + Skip per-feature geometries; the returned object will be a plain + ``DataFrame`` with no spatial information. The Water Data APIs + use camelCase ``skipGeometry`` in CQL2 queries. + bbox : list of numbers, optional + Only features whose geometry intersects the bounding box are + selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 + (longitude/latitude, west-south-east-north). + limit : int, optional + Page size; the maximum allowable value is 50000. Default + (``None``) requests the maximum allowable limit. This is a + per-page size, not a cap on the total result: a query matching more + rows than ``limit`` still returns every matching row across + multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object pertaining to the query. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # All time series and field measurements at a single surface-water site + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... monitoring_location_id="USGS-05407000" + ... ) + + >>> # Same, for a groundwater well — water-level and aquifer columns + >>> # are populated where the surface-water example has nulls + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... monitoring_location_id="USGS-375907091432201" + ... ) + + >>> # Every series in a single county, useful for area-of-interest workflows + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... state="Wisconsin", county_name="Dane County" + ... ) + + >>> # Inventory across multiple HUCs, restricted to streams and springs + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... hydrologic_unit_code=["11010008", "11010009"], + ... site_type=["Stream", "Spring"], + ... ) + + >>> # Discharge time series at three sites with at least one + >>> # observation in the past month + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... monitoring_location_id=[ + ... "USGS-07069000", + ... "USGS-07064000", + ... "USGS-07068000", + ... ], + ... end="P1M", + ... parameter_code="00060", + ... ) + + >>> # Two-step "what's available?" → "fetch it" workflow: + >>> # 1. inventory the sites in two HUCs + >>> hucs, _ = dataretrieval.waterdata.get_combined_metadata( + ... hydrologic_unit_code=["11010008", "11010009"], + ... site_type="Stream", + ... ) + >>> # 2. pull continuous discharge at every distinct site found + >>> sites = hucs["monitoring_location_id"].unique().tolist() + >>> df, md = dataretrieval.waterdata.get_continuous( + ... monitoring_location_id=sites, + ... parameter_code="00060", + ... time="P1D", + ... ) + + """ + service = "combined-metadata" + + # Resolve the unified `state` argument into the OGC `state_name` queryable. + args = _get_args( + _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} + ) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_field_measurements_metadata( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + parameter_name: str | Iterable[str] | None = None, + parameter_description: str | Iterable[str] | None = None, + begin: str | Iterable[str] | None = None, + end: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get field-measurement metadata: one row per (location, parameter) series. + + Each row describes a single field-measurement series — what parameter is + measured at the location, the period of record (``begin`` / ``end``), the + units, and so on — without returning the underlying observations + themselves. Use :func:`get_field_measurements` to fetch the values. + + This is the discrete-measurement analogue to + :func:`get_time_series_metadata` (which describes daily and continuous + series). It's primarily useful for inventory queries: "what + field-measurement parameters does this site have, and over what date + range?" + + See the OpenAPI reference for the full list of supported fields: + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements-metadata + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location, in + ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). + parameter_code : string or iterable of strings, optional + 5-digit parameter code. See + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + parameter_name : string or iterable of strings, optional + A human-understandable name corresponding to ``parameter_code``. + parameter_description : string or iterable of strings, optional + A human-readable description of what is being measured. + begin, end, last_modified : string, optional + Datetime fields that accept either an RFC 3339 datetime, an + interval (``"start/end"``, optionally half-bounded with ``..``), + or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See + :func:`get_time_series_metadata` for the full grammar. + properties : string or iterable of strings, optional + Subset of columns to return. Defaults to every available property. + skip_geometry : boolean, optional + Skip per-feature geometries; the returned object will be a plain + ``DataFrame`` with no spatial information. + bbox : list of numbers, optional + Only features whose geometry intersects the bounding box are + selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 + (longitude / latitude, west-south-east-north). + limit : int, optional + Page size; the maximum allowable value is 50000. Default + (``None``) requests the maximum allowable limit. This is a + per-page size, not a cap on the total result: a query matching more + rows than ``limit`` still returns every matching row across + multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object pertaining to the query. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # All field-measurement series at a surface-water site + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id="USGS-02238500" + ... ) + + >>> # Same, for a groundwater well + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id="USGS-375907091432201" + ... ) + + >>> # Multi-site, narrowed to two parameter codes + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id=[ + ... "USGS-451605097071701", + ... "USGS-263819081585801", + ... ], + ... parameter_code=["62611", "72019"], + ... ) + + >>> # Series modified in the last year — useful for incremental ETL + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id="USGS-375907091432201", + ... parameter_code="72019", + ... last_modified="P1Y", + ... ) + + """ + service = "field-measurements-metadata" + + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +__all__ = [ + "get_monitoring_locations", + "get_time_series_metadata", + "get_combined_metadata", + "get_field_measurements_metadata", +] diff --git a/dataretrieval/waterdata/nearest.py b/dataretrieval/waterdata/nearest.py index edf1a912..3ee0c063 100644 --- a/dataretrieval/waterdata/nearest.py +++ b/dataretrieval/waterdata/nearest.py @@ -11,7 +11,10 @@ import pandas as pd from dataretrieval.utils import BaseMetadata -from dataretrieval.waterdata.api import get_continuous +from dataretrieval.waterdata.time_series import get_continuous + +__all__ = ["get_nearest_continuous"] + OnTie = Literal["first", "last", "mean"] _VALID_ON_TIE: tuple[OnTie, ...] = get_args(OnTie) diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index cbaab057..041bfd51 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -23,10 +23,21 @@ from dataretrieval.ogc.filters import _quote_cql_str from dataretrieval.ogc.requests import _check_monitoring_location_id from dataretrieval.rdb import extract_rdb_comment, read_rdb -from dataretrieval.utils import HTTPX_DEFAULTS, _default_headers, _get +from dataretrieval.transport.http import ( + HTTPX_DEFAULTS, +) +from dataretrieval.transport.http import ( + default_headers as _default_headers, +) +from dataretrieval.transport.http import ( + get as _get, +) from .utils import BASE_URL +__all__ = ["get_ratings"] + + logger = logging.getLogger(__name__) STAC_URL = f"{BASE_URL}/stac/v0" diff --git a/dataretrieval/waterdata/reference.py b/dataretrieval/waterdata/reference.py new file mode 100644 index 00000000..ce95a155 --- /dev/null +++ b/dataretrieval/waterdata/reference.py @@ -0,0 +1,174 @@ +"""Reference-table and queryables discovery getters.""" + +from __future__ import annotations + +from typing import Any, get_args + +import pandas as pd + +from dataretrieval.ogc.schema import _check_ogc_requests +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.types import ( + METADATA_COLLECTIONS, +) +from dataretrieval.waterdata.utils import ( + get_ogc_data, +) + + +def get_reference_table( + collection: str, + limit: int | None = None, + query: dict[str, Any] | None = None, + max_rows: int | None = None, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get metadata reference tables for the USGS Water Data API. + + Reference tables provide the range of allowable values for parameter + arguments in the waterdata module. + + Parameters + ---------- + collection : string + One of the following options: "agency-codes", "altitude-datums", + "aquifer-codes", "aquifer-types", "coordinate-accuracy-codes", + "coordinate-datum-codes", "coordinate-method-codes", "counties", + "hydrologic-unit-codes", "medium-codes", "national-aquifer-codes", + "parameter-codes", "reliability-codes", "site-types", "states", + "statistic-codes", "topographic-codes", "time-zone-codes" + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + query: dictionary, optional + The optional query parameter can be used to pass a dictionary of + query parameters to the collection API call. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole table. Useful for cheaply + previewing large tables (e.g. ``hydrologic-unit-codes`` has ~125k + rows). Unlike ``limit`` (the per-page size), this bounds the total + result. The default (None) downloads every page. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. The primary metadata + of each reference table will show up in the first column, where + the name of the column is the singular form of the collection name, + separated by underscores (e.g. the "medium-codes" reference table + has a column called "medium_code", which contains all possible + medium code values). + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object including the URL request and query time. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get table of USGS parameter codes + >>> ref, md = dataretrieval.waterdata.get_reference_table( + ... collection="parameter-codes" + ... ) + + >>> # Get table of selected USGS parameter codes + >>> ref, md = dataretrieval.waterdata.get_reference_table( + ... collection="parameter-codes", + ... query={"id": "00001,00002"}, + ... ) + """ + valid_code_services = get_args(METADATA_COLLECTIONS) + if collection not in valid_code_services: + raise ValueError( + f"Invalid code service: '{collection}'. " + f"Valid options are: {valid_code_services}." + ) + + # Give the ID column the collection name, singularized and underscored. + if collection == "counties": + output_id = "county" + elif collection.endswith("s"): + output_id = collection[:-1].replace("-", "_") + else: + output_id = collection.replace("-", "_") + + query_args = dict(query) if query else {} + if limit is not None: + query_args["limit"] = limit + return get_ogc_data( + args=query_args, output_id=output_id, service=collection, max_rows=max_rows + ) + + +def get_queryables(collection: str) -> tuple[pd.DataFrame, BaseMetadata]: + """List the queryable properties of a Water Data API collection. + + Every OGC collection (``daily``, ``continuous``, ``monitoring-locations``, + ...) advertises the set of properties that can be filtered on -- exposed as + the typed keyword arguments of the matching ``get_*`` function, and usable + directly in a CQL2 ``filter``. This returns that set, so the available + filters can be discovered programmatically and monitored for upstream + additions. + + Parameters + ---------- + collection : string + The collection id, e.g. ``"daily"``, ``"continuous"``, + ``"monitoring-locations"``, or ``"time-series-metadata"``. See + :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` for the data + collections; reference collections (e.g. ``"parameter-codes"``) work + too. + + Returns + ------- + df : ``pandas.DataFrame`` + One row per queryable, sorted by name, with columns ``queryable`` (the + property name), ``type``, ``title``, and ``description``. + md : :class:`dataretrieval.utils.BaseMetadata` + Metadata describing the request (URL, query time, response headers). + + Raises + ------ + DataRetrievalError + On an HTTP error response (e.g. an unknown ``collection`` yields a 404), + the typed subclass for the status. + + Examples + -------- + .. doctest:: + :skipif: True # network + + >>> from dataretrieval import waterdata + >>> df, md = waterdata.get_queryables("daily") + >>> df.set_index("queryable").loc["state_name", "type"] + 'string' + """ + # The OGC queryables document is a JSON Schema whose ``properties`` map each + # filterable property name to a ``{title, type, description}`` definition. + body, response = _check_ogc_requests(endpoint=collection, req_type="queryables") + properties: dict[str, Any] = body.get("properties", {}) + df = pd.DataFrame( + [ + { + "queryable": name, + "type": prop.get("type"), + "title": prop.get("title"), + "description": (prop.get("description") or "").strip(), + } + for name, prop in sorted(properties.items()) + ], + columns=["queryable", "type", "title", "description"], + ) + return df, BaseMetadata(response) + + +__all__ = ["get_reference_table", "get_queryables"] diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py new file mode 100644 index 00000000..aaba15cf --- /dev/null +++ b/dataretrieval/waterdata/samples.py @@ -0,0 +1,432 @@ +"""Aquarius Samples API getters and wire-parameter policy.""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Iterable +from io import StringIO +from typing import Any, get_args +from urllib.parse import quote + +import httpx +import pandas as pd + +from dataretrieval.ogc.errors import _raise_for_non_200 +from dataretrieval.transport.http import ( + HTTPX_DEFAULTS, +) +from dataretrieval.transport.http import ( + default_headers as _default_headers, +) +from dataretrieval.transport.http import ( + get as _get, +) +from dataretrieval.utils import BaseMetadata, _attach_datetime_columns, to_str +from dataretrieval.waterdata.types import ( + CODE_SERVICES, + PROFILES, + SERVICES, +) +from dataretrieval.waterdata.utils import ( + SAMPLES_URL, + _accept_legacy_kwargs, + _check_profiles, + _get_args, +) + +logger = logging.getLogger(__name__) + + +def get_codes(code_service: CODE_SERVICES) -> tuple[pd.DataFrame, BaseMetadata]: + """Return codes from a Samples code service. + + Parameters + ---------- + code_service : string + One of the following options: "states", "counties", "countries", + "sitetype", "samplemedia", "characteristicgroup", "characteristics", + or "observedproperty" + + Returns + ------- + df : ``pandas.DataFrame`` + The requested code table. + md : :obj:`dataretrieval.utils.BaseMetadata` + Metadata for the query (URL, query time, response headers). + """ + valid_code_services = get_args(CODE_SERVICES) + if code_service not in valid_code_services: + raise ValueError( + f"Invalid code service: '{code_service}'. " + f"Valid options are: {valid_code_services}." + ) + + url = f"{SAMPLES_URL}/codeservice/{code_service}?mimeType=application%2Fjson" + + response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) + + _raise_for_non_200(response) + + data_dict = json.loads(response.text) + data_list = data_dict["data"] + + df = pd.DataFrame(data_list) + + return df, BaseMetadata(response) + + +def _get_samples_csv( + url: str, params: dict[str, Any], ssl_check: bool +) -> tuple[pd.DataFrame, httpx.Response]: + """Issue a Samples CSV request and parse the body into a DataFrame. + + Shared tail for the Samples getters: sends the GET with the standard + headers (including ``X-Api-Key``), raises a typed error on a non-200 + (consistent with the OGC/stats path) instead of a bare + ``HTTPStatusError``, and reads the CSV. The caller wraps the response + as metadata and applies any per-getter post-step. + """ + logger.debug("Request: %s", httpx.URL(url).copy_merge_params(params)) + response = _get( + url, + params=params, + verify=ssl_check, + headers=_default_headers(url), + **HTTPX_DEFAULTS, + ) + _raise_for_non_200(response) + df = pd.read_csv(StringIO(response.text), delimiter=",") + return df, response + + +# Map the public snake_case ``get_samples`` parameters to the camelCase query +# parameter names the Samples API expects on the wire. ``characteristic`` is +# already snake_case-compatible (single word) and is sent unchanged. The +# remaining snake_case params are bookkeeping (``service``/``profile``/ +# ``ssl_check``) and never reach the request. +_SAMPLES_PARAM_TO_API = { + "activity_media_name": "activityMediaName", + "activity_start_date_lower": "activityStartDateLower", + "activity_start_date_upper": "activityStartDateUpper", + "activity_type_code": "activityTypeCode", + "characteristic_group": "characteristicGroup", + "characteristic_user_supplied": "characteristicUserSupplied", + "bbox": "boundingBox", + "country_code": "countryFips", + "state_code": "stateFips", + "county_code": "countyFips", + "site_type_code": "siteTypeCode", + "site_type_name": "siteTypeName", + "usgs_pcode": "usgsPCode", + "hydrologic_unit": "hydrologicUnit", + "monitoring_location_id": "monitoringLocationIdentifier", + "organization_id": "organizationIdentifier", + "point_location_latitude": "pointLocationLatitude", + "point_location_longitude": "pointLocationLongitude", + "point_location_within_miles": "pointLocationWithinMiles", + "project_id": "projectIdentifier", + "record_identifier_user_supplied": "recordIdentifierUserSupplied", +} + +# Deprecated camelCase keyword names (the Samples-API spelling) accepted for +# backward compatibility, mapped to the new snake_case parameter names. Derived +# from ``_SAMPLES_PARAM_TO_API`` so the two never drift apart. +_SAMPLES_LEGACY_KWARGS = { + api_name: py_name for py_name, api_name in _SAMPLES_PARAM_TO_API.items() +} + + +@_accept_legacy_kwargs(_SAMPLES_LEGACY_KWARGS) +def get_samples( + ssl_check: bool = True, + service: SERVICES = "results", + profile: PROFILES = "fullphyschem", + activity_media_name: str | Iterable[str] | None = None, + activity_start_date_lower: str | None = None, + activity_start_date_upper: str | None = None, + activity_type_code: str | Iterable[str] | None = None, + characteristic_group: str | Iterable[str] | None = None, + characteristic: str | Iterable[str] | None = None, + characteristic_user_supplied: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + country_code: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type_name: str | Iterable[str] | None = None, + usgs_pcode: str | Iterable[str] | None = None, + hydrologic_unit: str | Iterable[str] | None = None, + monitoring_location_id: str | Iterable[str] | None = None, + organization_id: str | Iterable[str] | None = None, + point_location_latitude: float | None = None, + point_location_longitude: float | None = None, + point_location_within_miles: float | None = None, + project_id: str | Iterable[str] | None = None, + record_identifier_user_supplied: str | Iterable[str] | None = None, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Search Samples database for USGS water quality data. + This is a wrapper function for the Samples database API. All potential + filters are provided as arguments to the function, but please do not + populate all possible filters; leave as many as feasible with their default + value (None). This is important because overcomplicated web service queries + can bog down the database's ability to return an applicable dataset before + it times out. + + The web GUI for the Samples database can be found here: + https://waterdata.usgs.gov/download-samples/#dataProfile=site + + If you would like more details on feasible query parameters (complete with + examples), please visit the Samples database swagger docs, here: + https://api.waterdata.usgs.gov/samples-data/docs#/ + + Parameters + ---------- + ssl_check : bool, optional + Check the SSL certificate. + service : string + One of the available Samples services: "results", "locations", "activities", + "projects", or "organizations". Defaults to "results". + profile : string + One of the available profiles associated with a service. Options for each + service are: + results - "fullphyschem", "basicphyschem", + "fullbio", "basicbio", "narrow", + "resultdetectionquantitationlimit", + "labsampleprep", "count" + locations - "site", "count" + activities - "sampact", "actmetric", + "actgroup", "count" + projects - "project", "projectmonitoringlocationweight" + organizations - "organization", "count" + activity_media_name : string or iterable of strings, optional + Name or code indicating environmental medium in which sample was taken. + Call ``get_codes("samplemedia")`` for the valid inputs. + Example: "Water". (Samples API: ``activityMediaName``) + activity_start_date_lower : string, optional + The start date if using a date range. Takes the format YYYY-MM-DD. + The logic is inclusive, i.e. it will also return results that + match the date. If left as None, will pull all data on or before + ``activity_start_date_upper``, if populated. + (Samples API: ``activityStartDateLower``) + activity_start_date_upper : string, optional + The end date if using a date range. Takes the format YYYY-MM-DD. + The logic is inclusive, i.e. it will also return results that + match the date. If left as None, will pull all data after + ``activity_start_date_lower`` up to the most recent available results. + (Samples API: ``activityStartDateUpper``) + activity_type_code : string or iterable of strings, optional + Text code that describes type of field activity performed. + Example: "Sample-Routine, regular". (Samples API: ``activityTypeCode``) + characteristic_group : string or iterable of strings, optional + Characteristic group is a broad category of characteristics + describing one or more results. Call ``get_codes("characteristicgroup")`` + for the valid inputs. + Example: "Organics, PFAS" (Samples API: ``characteristicGroup``) + characteristic : string or iterable of strings, optional + Characteristic is a specific category describing one or more results. + Call ``get_codes("characteristics")`` for the valid inputs. + Example: "Suspended Sediment Discharge" (Samples API: ``characteristic``) + characteristic_user_supplied : string or iterable of strings, optional + A user supplied characteristic name describing one or more results. + (Samples API: ``characteristicUserSupplied``) + bbox : list of four floats, optional + Filters on the associated monitoring location's point location + by checking if it is located within the specified geographic area. + The logic is inclusive, i.e. it will include locations that overlap + with the edge of the bounding box. Values are separated by commas, + expressed in decimal degrees, NAD83, and longitudes west of Greenwich + are negative. The format is a list consisting of: + + * Western-most longitude + * Southern-most latitude + * Eastern-most longitude + * Northern-most latitude + + Example: [-92.8,44.2,-88.9,46.0] (Samples API: ``boundingBox``) + country_code : string or iterable of strings, optional + Example: "US" (United States) (Samples API: ``countryFips``) + state_code : string or iterable of strings, optional + Call ``get_codes("states")`` for the valid inputs. + Example: "US:15" (United States: Hawaii) (Samples API: ``stateFips``) + county_code : string or iterable of strings, optional + Call ``get_codes("counties")`` for the valid inputs. + Example: "US:15:001" (United States: Hawaii, Hawaii County) + (Samples API: ``countyFips``) + site_type_code : string or iterable of strings, optional + An abbreviation for a certain site type. Call ``get_codes("sitetype")`` + for the valid inputs. + Example: "GW" (Groundwater site) (Samples API: ``siteTypeCode``) + site_type_name : string or iterable of strings, optional + A full name for a certain site type. Call ``get_codes("sitetype")`` + for the valid inputs. + Example: "Well" (Samples API: ``siteTypeName``) + usgs_pcode : string or iterable of strings, optional + 5-digit number used in the US Geological Survey computerized + data system, National Water Information System (NWIS), to + uniquely identify a specific constituent (the ``parameterCode`` column + of ``get_codes("characteristics")``). + Example: "00060" (Discharge, cubic feet per second) + (Samples API: ``usgsPCode``) + hydrologic_unit : string or iterable of strings, optional + Max 12-digit number used to describe a hydrologic unit. + Example: "070900020502" (Samples API: ``hydrologicUnit``) + monitoring_location_id : string or iterable of strings, optional + A monitoring location identifier has two parts: the agency code + and the location number, separated by a dash (-). + Example: "USGS-040851385" + (Samples API: ``monitoringLocationIdentifier``) + organization_id : string or iterable of strings, optional + Designator used to uniquely identify a specific organization. + Currently only accepting the organization "USGS". + (Samples API: ``organizationIdentifier``) + point_location_latitude : float, optional + Latitude for a point/radius query (decimal degrees). Must be used + with ``point_location_longitude`` and ``point_location_within_miles``. + (Samples API: ``pointLocationLatitude``) + point_location_longitude : float, optional + Longitude for a point/radius query (decimal degrees). Must be used + with ``point_location_latitude`` and ``point_location_within_miles``. + (Samples API: ``pointLocationLongitude``) + point_location_within_miles : float, optional + Radius for a point/radius query. Must be used with + ``point_location_latitude`` and ``point_location_longitude``. + (Samples API: ``pointLocationWithinMiles``) + project_id : string or iterable of strings, optional + Designator used to uniquely identify a data collection project. Project + identifiers are specific to an organization (e.g. USGS). + Example: "ZH003QW03" (Samples API: ``projectIdentifier``) + record_identifier_user_supplied : string or iterable of strings, optional + Internal AQS record identifier that returns 1 entry. Only available + for the "results" service. + (Samples API: ``recordIdentifierUserSupplied``) + + Returns + ------- + df : ``pandas.DataFrame`` + Formatted data returned from the API query. For each + ``Date`` / ``Time`` / ``TimeZone`` triplet in + the response (e.g. ``Activity_StartDate``, ``Activity_StartTime``, + ``Activity_StartTimeZone``), an additional ``DateTime`` column + is appended holding a UTC ``Timestamp`` derived from the three. The + original Date/Time/TimeZone columns are left intact; rows whose + timezone abbreviation is not recognized resolve to ``NaT``. Rows are + sorted by ``Activity_StartDateTime`` when present (the API's default + order is unstable). + md : :obj:`dataretrieval.utils.BaseMetadata` + Custom ``dataretrieval`` metadata object pertaining to the query. + + Examples + -------- + .. code:: + + >>> # Get PFAS results within a bounding box + >>> df, md = dataretrieval.waterdata.get_samples( + ... bbox=[-90.2, 42.6, -88.7, 43.2], + ... characteristic_group="Organics, PFAS", + ... ) + + >>> # Get all activities for the Commonwealth of Virginia over a date range + >>> df, md = dataretrieval.waterdata.get_samples( + ... service="activities", + ... profile="sampact", + ... activity_start_date_lower="2023-10-01", + ... activity_start_date_upper="2024-01-01", + ... state_code="US:51", + ... ) + + >>> # Get all pH samples for two sites in Utah + >>> df, md = dataretrieval.waterdata.get_samples( + ... monitoring_location_id=[ + ... "USGS-393147111462301", + ... "USGS-393343111454101", + ... ], + ... usgs_pcode="00400", + ... ) + + """ + + _check_profiles(service, profile) + + # Build argument dictionary, omitting None values. Parameters are the + # public snake_case names here; translate them to the camelCase names the + # Samples API expects just before building the request. + args = _get_args(locals(), exclude={"ssl_check", "profile"}) + params = {_SAMPLES_PARAM_TO_API.get(key, key): value for key, value in args.items()} + + params.update({"mimeType": "text/csv"}) + + if "boundingBox" in params: + params["boundingBox"] = to_str(params["boundingBox"]) + + url = f"{SAMPLES_URL}/{service}/{profile}" + + df, response = _get_samples_csv(url, params, ssl_check) + df = _attach_datetime_columns(df) + + return df, BaseMetadata(response) + + +@_accept_legacy_kwargs({"monitoringLocationIdentifier": "monitoring_location_id"}) +def get_samples_summary( + monitoring_location_id: str, + ssl_check: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get a summary of discrete water-quality samples at a single monitoring location. + + Wraps the Samples database summary service described at + https://api.waterdata.usgs.gov/samples-data/docs. The service returns one + row per (characteristic group, characteristic, user-supplied characteristic) + combination with result and activity counts and the first / most recent + activity dates — useful for taking inventory of what discrete-sample data + exists at a site before pulling the underlying observations with + :func:`get_samples`. + + The summary service is single-site only: it accepts exactly one monitoring + location per request. + + Parameters + ---------- + monitoring_location_id : string + A monitoring location identifier has two parts, separated by a dash + (``-``): the agency code and the location number. Examples: + ``"USGS-040851385"``, ``"AZ014-320821110580701"``, + ``"CAX01-15304600"``. Bare location numbers without an agency prefix + are accepted by the service but return an empty result, so a prefix + is effectively required. (Samples API: ``monitoringLocationIdentifier``) + ssl_check : bool, optional + Check the SSL certificate. Default is True. + + Returns + ------- + df : ``pandas.DataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + Custom ``dataretrieval`` metadata object pertaining to the query. + + Examples + -------- + .. code:: + + >>> # What discrete-sample data is available at this site? + >>> df, md = dataretrieval.waterdata.get_samples_summary( + ... monitoring_location_id="USGS-04074950" + ... ) + + """ + if not isinstance(monitoring_location_id, str): + raise TypeError( + "monitoring_location_id must be a string; the Samples " + "summary service accepts exactly one monitoring location per " + f"request, got {type(monitoring_location_id).__name__}." + ) + + url = f"{SAMPLES_URL}/summary/{quote(monitoring_location_id, safe='')}" + params = {"mimeType": "text/csv"} + + df, response = _get_samples_csv(url, params, ssl_check) + + return df, BaseMetadata(response) + + +__all__ = ["get_codes", "get_samples", "get_samples_summary"] diff --git a/dataretrieval/waterdata/stats.py b/dataretrieval/waterdata/stats.py index abba5deb..9798b16d 100644 --- a/dataretrieval/waterdata/stats.py +++ b/dataretrieval/waterdata/stats.py @@ -3,8 +3,9 @@ Wraps ``https://api.waterdata.usgs.gov/statistics/v0`` — the daily-statistics service (period-of-record and date-range normals/intervals). This is a *separate*, non-OGC API: it has no chunkable multi-value axes, so it drives -:func:`engine._paginate` directly through a blocking portal rather than going -through ``multi_value_chunked``. The typed getters ``get_stats_por`` and +:func:`dataretrieval.transport.pagination.paginate` through the shared sync +bridge rather than going through ``multi_value_chunked``. The typed getters +``get_stats_por`` and ``get_stats_date_range`` in :mod:`dataretrieval.waterdata.api` call :func:`get_data` here. """ @@ -16,19 +17,22 @@ import httpx import pandas as pd -from dataretrieval.ogc.engine import ( - _paginate, - _run_sync, -) +from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.shaping import ( _CRS, GEOPANDAS, _attach_coordinates, _empty_feature_frame, ) -from dataretrieval.utils import BaseMetadata, _default_headers +from dataretrieval.transport.http import default_headers +from dataretrieval.transport.pagination import paginate +from dataretrieval.transport.sync import run_sync +from dataretrieval.utils import BaseMetadata from dataretrieval.waterdata.utils import BASE_URL +__all__ = ["get_data"] + + # ``_handle_nesting``'s geopandas branch calls ``gpd.GeoDataFrame.from_features`` # directly, so this module needs its own bound ``gpd`` name. Import it under the # same guard the engine uses; when geopandas is absent ``gpd`` is left unbound @@ -220,7 +224,7 @@ def get_data( to the specified parameters. The stats path doesn't go through ``multi_value_chunked`` (its query - shape has no chunkable list axes), so it drives :func:`engine._paginate` + shape has no chunkable list axes), so it drives transport pagination directly through an ``anyio`` blocking portal. The portal runs the pagination loop in a short-lived worker thread, so this works whether or not the caller is already inside an event loop. @@ -251,7 +255,8 @@ def get_data( Raises ------ DataRetrievalError - The typed subclass for an HTTP error response (see :func:`engine._paginate`); + The typed subclass for an HTTP error response (see + :func:`transport.pagination.paginate`); or :class:`~dataretrieval.exceptions.NetworkError` if the initial request can't reach the service (timeout / DNS), the ``httpx`` exception chained on ``__cause__``. @@ -261,7 +266,7 @@ def get_data( req = httpx.Request( method="GET", url=url, - headers=_default_headers(url), + headers=default_headers(url), params=args, ) method = req.method @@ -282,14 +287,15 @@ async def follow_up(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: ) async def _run() -> tuple[pd.DataFrame, httpx.Response]: - return await _paginate( + return await paginate( req, parse_response=parse_response, follow_up=follow_up, client=client, + raise_for_status=_raise_for_non_200, ) - df, response = _run_sync(_run, service=service) + df, response = run_sync(_run, service=service, error_url=url) if expand_percentiles: df = _expand_percentiles(df) diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py new file mode 100644 index 00000000..b5fb78a9 --- /dev/null +++ b/dataretrieval/waterdata/time_series.py @@ -0,0 +1,1197 @@ +"""Time-series observation and statistics getters.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc.filters import FILTER_LANG +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata import stats +from dataretrieval.waterdata.utils import ( + _get_args, + _with_state, + get_ogc_data, +) + + +def get_daily( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + daily_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Daily data provide one data value to represent water conditions for the + day. + + Throughout much of the history of the USGS, the primary water data available + was daily data collected manually at the monitoring location once each day. + With improved availability of computer storage and automated transmission of + data, the daily data published today are generally a statistical summary or + metric of the continuous data collected each day, such as the daily mean, + minimum, or maximum value. Daily data are automatically calculated from the + continuous data of the same parameter code and are described by parameter + code and a statistic code. These data have also been referred to as “daily + values” or “DV”. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter + codes and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. + Available options are: geometry, id, time_series_id, + monitoring_location_id, parameter_code, statistic_id, time, value, + unit_of_measure, approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + daily_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + Only features that have a last_modified that intersects the value of + datetime are selected. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get daily flow data from a single site + >>> # over a yearlong period + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", + ... ) + + >>> # Quick "show me the last week" idiom (ISO 8601 duration) + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... time="P7D", + ... ) + + >>> # Get approved daily flow data from multiple sites + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], + ... approval_status="Approved", + ... time="2024-01-01/..", + ... ) + + >>> # Pull only rows whose underlying record was refreshed in the + >>> # last 7 days — handy for incremental ETL polling + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... last_modified="P7D", + ... ) + + >>> # Chain queries: pull all stream sites in a state, then their + >>> # daily discharge for the last week. The site list can be hundreds + >>> # of values long — the request is transparently chunked across + >>> # multiple sub-requests so the URL stays under the server's byte + >>> # limit. Combined output looks like a single query. + >>> sites_df, _ = dataretrieval.waterdata.get_monitoring_locations( + ... state="Ohio", + ... site_type="Stream", + ... ) + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id=sites_df["monitoring_location_id"].tolist(), + ... parameter_code="00060", + ... time="P7D", + ... ) + """ + service = "daily" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_continuous( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + continuous_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + time: str | Iterable[str] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """ + Continuous data provide instantaneous water conditions. + + This is an early version of the continuous endpoint that is feature-complete + and is being made available for limited use. Geometries are not included + with the continuous endpoint. If the "time" input is left blank, the service + will return the most recent year of measurements. Users may request no more + than three years of data with each function call. + + Continuous data are collected at a high frequency, typically 15-minute + intervals. Depending on the specific monitoring location, the data may be + transmitted automatically via telemetry and be available on WDFN within + minutes of collection, while other times the delivery of data may be delayed + if the monitoring location does not have the capacity to automatically + transmit data. Continuous data are described by parameter name and + parameter code (pcode). These data might also be referred to as + "instantaneous values" or "IV". + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter + codes and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Continuous data are nearly always associated with statistic id + 00011. Using a different code (such as 00003 for mean) will + typically return no results. A complete list of codes and their + descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. + Available options are: geometry, id, time_series_id, + monitoring_location_id, parameter_code, statistic_id, time, value, + unit_of_measure, approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + continuous_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + Only features that have a last_modified that intersects the value of + datetime are selected. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 10000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get instantaneous gage height data from a + >>> # single site from a single year + >>> df, md = dataretrieval.waterdata.get_continuous( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00065", + ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", + ... ) + + >>> # Pull several disjoint time windows in one call via a CQL + >>> # ``filter``. See ``dataretrieval.ogc.filters`` for the + >>> # full grammar, auto-chunking, and pitfalls. + >>> df, md = dataretrieval.waterdata.get_continuous( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... filter=( + ... "(time >= '2023-06-01T12:00:00Z' " + ... "AND time <= '2023-06-01T13:00:00Z') " + ... "OR (time >= '2023-06-15T12:00:00Z' " + ... "AND time <= '2023-06-15T13:00:00Z')" + ... ), + ... filter_lang="cql-text", + ... ) + """ + service = "continuous" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_latest_continuous( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + latest_continuous_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """This endpoint provides the most recent observation for each time series + of continuous data. Continuous data are collected via automated sensors + installed at a monitoring location. They are collected at a high frequency + and often at a fixed 15-minute interval. Depending on the specific monitoring + location, the data may be transmitted automatically via telemetry and be + available on WDFN within minutes of collection, while other times the delivery + of data may be delayed if the monitoring location does not have the capacity to + automatically transmit data. Continuous data are described by parameter name + and parameter code. These data might also be referred to as "instantaneous + values" or "IV". + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, id, time_series_id, monitoring_location_id, + parameter_code, statistic_id, time, value, unit_of_measure, + approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + latest_continuous_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get latest flow data from a single site + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id="USGS-02238500", parameter_code="00060" + ... ) + + >>> # Restrict to the last 7 days; sites with no observation in that + >>> # window are dropped instead of returned with stale values + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... time="P7D", + ... ) + + >>> # Pull only rows whose underlying record was refreshed in the + >>> # last 7 days, across multiple sites and parameters + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id=["USGS-451605097071701", "USGS-14181500"], + ... parameter_code=["00060", "72019"], + ... last_modified="P7D", + ... ) + + >>> # Get latest continuous measurements for multiple sites + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] + ... ) + """ + service = "latest-continuous" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_latest_daily( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + latest_daily_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Daily data provide one data value to represent water conditions for the + day. + + Throughout much of the history of the USGS, the primary water data available + was daily data collected manually at the monitoring location once each day. + With improved availability of computer storage and automated transmission of + data, the daily data published today are generally a statistical summary or + metric of the continuous data collected each day, such as the daily mean, + minimum, or maximum value. Daily data are automatically calculated from the + continuous data of the same parameter code and are described by parameter + code and a statistic code. These data have also been referred to as “daily + values” or “DV”. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, id, time_series_id, monitoring_location_id, + parameter_code, statistic_id, time, value, unit_of_measure, + approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + latest_daily_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get most recent daily flow data from a single site + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id="USGS-02238500", parameter_code="00060" + ... ) + + >>> # Restrict to rows whose underlying record was refreshed in the + >>> # last 7 days + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... last_modified="P7D", + ... ) + + >>> # Multi-site, multi-parameter — discharge and water temperature + >>> # at two sites in a single round-trip + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id=["USGS-01491000", "USGS-01645000"], + ... parameter_code=["00060", "00010"], + ... ) + + >>> # Get most recent daily measurements for two sites + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] + ... ) + """ + service = "latest-daily" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_stats_por( + approval_status: str | None = None, + computation_type: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, + monitoring_location_id: str | Iterable[str] | None = None, + page_size: int = 1000, + parent_time_series_id: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type_name: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + normal_type: str | None = None, + expand_percentiles: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get day-of-year and month-of-year water data statistics from the + USGS Water Data API. + This service (called the "observationNormals" endpoint on api.waterdata.usgs.gov) + provides endpoints for access to computations on the historical record regarding + water conditions, including minimum, maximum, mean, median, and percentiles for + day of year and month of year. For more information regarding the calculation of + statistics and other details, please visit the Statistics documentation page: + https://waterdata.usgs.gov/statistics-documentation/. + + Note: This API is under active beta development and subject to + change. Improved handling of significant figures will be + addressed in a future release. + + Parameters + ---------- + approval_status: string, optional + Whether to include approved and/or provisional observations. + At this time, only approved observations are returned. + computation_type: string, optional + Desired statistical computation method. Available values are: + arithmetic_mean, maximum, median, minimum, percentile. + country_code: string, optional + Country query parameter. API defaults to "US". + state: string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit + ANSI/FIPS code ("55"). + state_code: string, optional + State query parameter. Takes the format "US:XX", where XX is + the two-digit state code. API defaults to "US:42" (Pennsylvania). + county_code: string, optional + County query parameter. Takes the format "US:XX:YYY", where XX is + the two-digit state code and YYY is the three-digit county code. + API defaults to "US:42:103" (Pennsylvania, Pike County). + start_date: string or datetime, optional + Start day for the query in the month-day format (MM-DD). + end_date: string or datetime, optional + End day for the query in the month-day format (MM-DD). + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + page_size : int, optional + The number of results to return per page, where one result represents a + monitoring location. The default is 1000. + parent_time_series_id: string, optional + The parent_time_series_id returns statistics tied to a + particular database entry. + site_type_code: string, optional + Site type code query parameter. + A list of valid site type codes is available at: + https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + Example: "GW" (Groundwater site) + site_type_name: string, optional + Site type name query parameter. + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + normal_type : string, optional + Filter the returned normals to a single period. If unspecified + (default), all matching data are returned. Available values: + "DOY" (day-of-year) and "MOY" (month-of-year). + expand_percentiles : boolean + Percentile data for a given day of year or month of year by default + are returned from the service as lists of string values and percentile + thresholds in the "values" and "percentiles" columns, respectively. + When `expand_percentiles` is set to True (default), each value and + percentile threshold specific to a computation id are returned as + individual rows in the dataframe, with the value reported in the + "value" column and the corresponding percentile reported in a + "percentile" column (and the "values" and "percentiles" columns + are removed). Missing percentile values expressed as 'nan' in the + list of string values are removed from the dataframe to save space. + Setting `expand_percentiles` to False retains the "values" and + "percentiles" columns produced by the service. Including + both 'percentiles' and one or more other statistics ('median', + 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` + argument will return both the "values" column, containing the list + of percentile threshold values, and a "value" column, containing + the singular summary value for the other statistics. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object. + + Examples + -------- + .. code:: + + >>> # Get daily, monthly, and annual percentiles for streamflow at + >>> # a monitoring location of interest + >>> df, md = dataretrieval.waterdata.get_stats_por( + ... monitoring_location_id="USGS-05114000", + ... parameter_code="00060", + ... computation_type="percentile", + ... ) + + >>> # Get all daily and monthly statistics for the month of January + >>> # over the entire period of record for streamflow and gage height + >>> # at a monitoring location of interest + >>> df, md = dataretrieval.waterdata.get_stats_por( + ... monitoring_location_id="USGS-05114000", + ... parameter_code=["00060", "00065"], + ... start_date="01-01", + ... end_date="01-31", + ... ) + """ + # Build argument dictionary, omitting None values + params = _get_args( + _with_state(locals(), to="fips_us", into="state_code"), + exclude={"expand_percentiles"}, + ) + + return stats.get_data( + args=params, service="observationNormals", expand_percentiles=expand_percentiles + ) + + +def get_stats_date_range( + approval_status: str | None = None, + computation_type: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, + monitoring_location_id: str | Iterable[str] | None = None, + page_size: int = 1000, + parent_time_series_id: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type_name: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + interval_type: str | Iterable[str] | None = None, + expand_percentiles: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get monthly and annual water data statistics from the USGS Water Data API. + This service (called the "observationIntervals" endpoint on api.waterdata.usgs.gov) + provides endpoints for access to computations on the historical record regarding + water conditions, including minimum, maximum, mean, median, and percentiles for + month-year, and water/calendar years. For more information regarding the calculation + of statistics and other details, please visit the Statistics documentation page: + https://waterdata.usgs.gov/statistics-documentation/. + + Note: This API is under active beta development and subject to + change. Improved handling of significant figures will be + addressed in a future release. + + Parameters + ---------- + approval_status: string, optional + Whether to include approved and/or provisional observations. + At this time, only approved observations are returned. + computation_type: string, optional + Desired statistical computation method. Available values are: + arithmetic_mean, maximum, median, minimum, percentile. + country_code: string, optional + Country query parameter. API defaults to "US". + state: string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit + ANSI/FIPS code ("55"). + state_code: string, optional + State query parameter. Takes the format "US:XX", where XX is + the two-digit state code. API defaults to "US:42" (Pennsylvania). + county_code: string, optional + County query parameter. Takes the format "US:XX:YYY", where XX is + the two-digit state code and YYY is the three-digit county code. + API defaults to "US:42:103" (Pennsylvania, Pike County). + start_date: string or datetime, optional + Start date for the query in the year-month-day format + (YYYY-MM-DD). + end_date: string or datetime, optional + End date for the query in the year-month-day format + (YYYY-MM-DD). + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + page_size : int, optional + The number of results to return per page, where one result represents a + monitoring location. The default is 1000. + parent_time_series_id: string, optional + The parent_time_series_id returns statistics tied to a + particular database entry. + site_type_code: string, optional + Site type code query parameter. + You can see a list of valid site type codes here: + https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + Example: "GW" (Groundwater site) + site_type_name: string, optional + Site type name query parameter. + You can see a list of valid site type names here: + https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + Example: "Well" + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + interval_type : string or iterable of strings, optional + Filter the returned intervals to one or more periods. If unspecified + (default), all matching data are returned. Available values: + "M" (month), "CY" (calendar year), and "WY" (water year). + expand_percentiles : boolean + Percentile data for a given day of year or month of year by default + are returned from the service as lists of string values and percentile + thresholds in the "values" and "percentiles" columns, respectively. + When `expand_percentiles` is set to True (default), each value and + percentile threshold specific to a computation id are returned as + individual rows in the dataframe, with the value reported in the + "value" column and the corresponding percentile reported in a + "percentile" column (and the "values" and "percentiles" columns + are removed). Missing percentile values expressed as 'nan' in the + list of string values are removed from the dataframe to save space. + Setting `expand_percentiles` to False retains the "values" and + "percentiles" columns produced by the service. Including + both 'percentiles' and one or more other statistics ('median', + 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` + argument will return both the "values" column, containing the list + of percentile threshold values, and a "value" column, containing + the singular summary value for the other statistics. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object. + + Examples + -------- + .. code:: + + >>> # Get monthly and yearly medians for streamflow at streams in Rhode Island + >>> # from calendar year 2024. + >>> df, md = dataretrieval.waterdata.get_stats_date_range( + ... state="RI", # Rhode Island (postal code, name, or FIPS all work) + ... parameter_code="00060", + ... site_type_code="ST", + ... start_date="2024-01-01", + ... end_date="2024-12-31", + ... computation_type="median", + ... ) + + >>> # Get monthly and yearly minimum and maximums for gage height at + >>> # a monitoring location of interest + >>> df, md = dataretrieval.waterdata.get_stats_date_range( + ... monitoring_location_id="USGS-05114000", + ... parameter_code="00065", + ... computation_type=["minimum", "maximum"], + ... ) + """ + # Build argument dictionary, omitting None values + params = _get_args( + _with_state(locals(), to="fips_us", into="state_code"), + exclude={"expand_percentiles"}, + ) + + return stats.get_data( + args=params, + service="observationIntervals", + expand_percentiles=expand_percentiles, + ) + + +__all__ = [ + "get_daily", + "get_continuous", + "get_latest_continuous", + "get_latest_daily", + "get_stats_por", + "get_stats_date_range", +] diff --git a/dataretrieval/waterdata/types.py b/dataretrieval/waterdata/types.py index 20753d3f..022627d0 100644 --- a/dataretrieval/waterdata/types.py +++ b/dataretrieval/waterdata/types.py @@ -1,5 +1,15 @@ from typing import Literal +__all__ = [ + "CODE_SERVICES", + "METADATA_COLLECTIONS", + "SERVICES", + "WATERDATA_SERVICES", + "PROFILES", + "PROFILE_LOOKUP", +] + + CODE_SERVICES = Literal[ "characteristicgroup", "characteristics", diff --git a/dataretrieval/wateruse.py b/dataretrieval/wateruse.py index 53de7597..d329bd38 100644 --- a/dataretrieval/wateruse.py +++ b/dataretrieval/wateruse.py @@ -12,11 +12,9 @@ (:mod:`dataretrieval.ngwmn`), the NWDC is a plain CSV REST service rather than an OGC API Features collection. This module supplies the NWDC-specific bits — request building, CSV parsing, the ``Link``-header cursor, and the ``{detail}`` -error envelope — but reuses the OGC engine's generic, API-agnostic pagination -and sync-from-async plumbing (:func:`~dataretrieval.ogc.engine._paginate` and -:func:`~dataretrieval.ogc.engine._run_sync`) rather than re-implementing it. It -follows the same conventions: shared request headers -(:func:`~dataretrieval.utils._default_headers`), the typed +error envelope — and uses the API-neutral transport layer for cursor pagination, +response aggregation, client lifecycle, and sync-from-async dispatch. It follows +the same conventions: host-scoped request headers, the typed :class:`~dataretrieval.exceptions.DataRetrievalError` taxonomy, and a ``(DataFrame, BaseMetadata)`` return. @@ -53,17 +51,27 @@ from dataretrieval.codes.states import to_state from dataretrieval.exceptions import DataRetrievalError -from dataretrieval.ogc.combining import _combine_chunk_frames, _combine_chunk_responses -from dataretrieval.ogc.engine import _paginate, _run_sync -from dataretrieval.utils import ( - HTTPX_ASYNC_DEFAULTS, - BaseMetadata, - _default_headers, - _raise_for_status, - to_str, +from dataretrieval.transport.combining import ( + _combine_chunk_frames, + _combine_chunk_responses, ) +from dataretrieval.transport.http import default_headers, open_async_client +from dataretrieval.transport.pagination import paginate +from dataretrieval.transport.retry import RetryPolicy, retry_async +from dataretrieval.transport.sync import run_sync +from dataretrieval.utils import BaseMetadata, _raise_for_status, to_str + +__all__ = [ + "get_wateruse", + "WATERUSE_URL", + "MODELS", + "TIME_RESOLUTIONS", + "MAX_CONCURRENT_REQUESTS", +] + WATERUSE_URL = "https://api.water.usgs.gov/nwaa-data/data" +_WATERUSE_HOST = httpx.URL(WATERUSE_URL).host #: Water-use models (categories) served by the NWDC. The catalog at #: https://water.usgs.gov/nwaa-data/ lists the variables available within each. @@ -222,9 +230,9 @@ def get_wateruse( base_params = {k: v for k, v in base_params.items() if v is not None} # The NWDC queries one location per request, so fan a multi-value selector - # out into one request per location, each paginated by the OGC engine's - # shared pager (``_paginate``), and concatenate the results. - headers = _default_headers(WATERUSE_URL) + # out into one request per location, each handled by shared transport + # pagination, and concatenate the results. + headers = default_headers(WATERUSE_URL) requests = [ httpx.Request( "GET", @@ -238,7 +246,7 @@ def get_wateruse( # even inside an already-running event loop (e.g. a Jupyter notebook). # ``error_url`` is the host reported in any connection-error message (this # module builds its own requests, so it has no OGC request-builder base). - df, response = _run_sync( + df, response = run_sync( lambda: _fan_out(requests, headers, ssl_check), service="wateruse", error_url=WATERUSE_URL, @@ -330,8 +338,8 @@ async def _fan_out( ) -> tuple[pd.DataFrame, httpx.Response]: """Fetch every request (each paginated) concurrently over one shared client. - Each request is paginated by the engine's - :func:`~dataretrieval.ogc.engine._paginate` with NWDC strategies: parse a CSV + Each request is paginated by :func:`dataretrieval.transport.pagination.paginate` + with NWDC strategies: parse a CSV page and read its ``Link`` header cursor (``parse``), follow that cursor (``follow``), and raise the typed error carrying the NWDC ``detail`` (``raise_for_status``). Concurrency is bounded by a semaphore at @@ -349,22 +357,29 @@ async def follow(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: def raise_for_status(response: httpx.Response) -> None: _raise_for_status(response, detail_from=_nwdc_error_detail) - async with httpx.AsyncClient(verify=ssl_check, **HTTPX_ASYNC_DEFAULTS) as client: + policy = RetryPolicy.from_env() + async with open_async_client(verify=ssl_check) as client: semaphore = asyncio.Semaphore(max(1, MAX_CONCURRENT_REQUESTS)) async def _one(request: httpx.Request) -> tuple[pd.DataFrame, httpx.Response]: - async with semaphore: - return await _paginate( - request, - parse_response=parse, - follow_up=follow, - client=client, - raise_for_status=raise_for_status, - ) + async def attempt() -> tuple[pd.DataFrame, httpx.Response]: + async with semaphore: + return await paginate( + request, + parse_response=parse, + follow_up=follow, + client=client, + raise_for_status=raise_for_status, + ) + + # Retry an initial transient after releasing the semaphore. A + # later-page failure is intentionally wrapped by ``paginate`` and + # propagates instead of restarting a partially completed walk. + return await retry_async(attempt, policy) results = await asyncio.gather(*(_one(req) for req in requests)) - # Reuse the engine's combine helpers: drop empty frames and concat, and fold + # Reuse the transport combine helpers: drop empty frames and concat, and fold # the per-location responses into one (headers from the response with the # lowest reported remaining quota plus summed response durations), keeping # the first request's URL as the query identity. @@ -399,7 +414,19 @@ def _next_page_url(response: httpx.Response) -> str | None: url = response.links.get("next", {}).get("url") if not url: return None - return str(url).replace("https://water.usgs.gov", "https://api.water.usgs.gov", 1) + normalized = str(url).replace( + "https://water.usgs.gov", "https://api.water.usgs.gov", 1 + ) + try: + host = httpx.URL(normalized).host + except (httpx.InvalidURL, TypeError) as exc: + raise RuntimeError("Refusing invalid Water Use next-page URL") from exc + if host != _WATERUSE_HOST: + raise RuntimeError( + f"Refusing to follow cross-host Water Use next-page URL: " + f"{host} != {_WATERUSE_HOST}" + ) + return normalized def _nwdc_error_detail(response: httpx.Response) -> str | None: diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index ffbaee91..a64d4dbf 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -17,7 +17,23 @@ import pandas as pd -from .utils import BaseMetadata, _attach_datetime_columns, query +from .utils import BaseMetadata, _attach_datetime_columns, _query_with_retry + +__all__ = [ + "get_results", + "what_sites", + "what_organizations", + "what_projects", + "what_activities", + "what_detection_limits", + "what_habitat_metrics", + "what_project_weights", + "what_activity_metrics", + "wqp_url", + "wqx3_url", + "WQP_Metadata", +] + if TYPE_CHECKING: import httpx @@ -179,7 +195,7 @@ def get_results( if legacy is not True and profile is None: kwargs["dataProfile"] = "fullPhysChem" - response = query(url, kwargs, delimiter=";", ssl_check=ssl_check) + response = _query_with_retry(url, kwargs, delimiter=";", ssl_check=ssl_check) df = _read_wqp_csv(response.text) df = _attach_datetime_columns(df) @@ -208,7 +224,9 @@ def _what( else: url = _legacy_only_url(service, legacy=legacy) - response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check) + response = _query_with_retry( + url, payload=kwargs, delimiter=";", ssl_check=ssl_check + ) df = _read_wqp_csv(response.text) return df, WQP_Metadata(response, **kwargs) diff --git a/docs/source/architecture/decisions/0003-dependency-direction.rst b/docs/source/architecture/decisions/0003-dependency-direction.rst index 3f7d8ec9..32c14131 100644 --- a/docs/source/architecture/decisions/0003-dependency-direction.rst +++ b/docs/source/architecture/decisions/0003-dependency-direction.rst @@ -18,13 +18,15 @@ unintended cross-package contracts. Decision -------- -Dependencies point from public facades to service/protocol adapters and then to -stable shared policy and third-party infrastructure. In particular: +Dependencies point from public facades to service/protocol adapters, then to +API-neutral transport and stable policy, and finally to third-party +infrastructure. In particular: - ``dataretrieval.exceptions`` is a runtime-dependency-light leaf. - ``dataretrieval.ogc`` must not import Water Data, NGWMN, Water Use, or NWIS. - Modern modules must not import deprecated NWIS. -- New non-OGC services must not obtain generic transport behavior by importing +- API-neutral transport must not import OGC modules or service adapters. +- Non-OGC services must obtain generic execution behavior from transport, not private OGC implementation symbols. Underscore-prefixed symbols remain implementation details even when existing @@ -49,8 +51,8 @@ second copy of that mutable inventory. Focused fitness functions verify the current boundaries: NGWMN's only OGC dependency is the facade, ``waterdata.utils`` does not bulk re-export private -OGC helpers, ``ogc.shaping`` does not depend on ``ogc.engine``, and the full OGC -runtime graph is acyclic. +OGC helpers, ``ogc.shaping`` does not depend on ``ogc.engine``, Water Use has +no OGC dependency, and both the OGC and transport runtime graphs are acyclic. The exact allowlist should shrink as private seams move. Any growth requires explicit architecture review, and a change to the dependency policy requires diff --git a/docs/source/architecture/decisions/0004-error-retry-resume.rst b/docs/source/architecture/decisions/0004-error-retry-resume.rst index 50c9e3d3..d05d7522 100644 --- a/docs/source/architecture/decisions/0004-error-retry-resume.rst +++ b/docs/source/architecture/decisions/0004-error-retry-resume.rst @@ -28,9 +28,10 @@ cancellation. OGC fan-out retains completed subrequests and raises a typed ``ChunkInterrupted`` with a handle that resumes only missing work. Fatal or unknown failures are not disguised as resumable transients. -This decision does not assert that every upstream API supports pagination, -chunking, or resume. Those capabilities remain explicit per service until a -shared API-neutral transport contract is introduced. +The shared transport layer supplies bounded retry and callback-driven cursor +pagination, but each adapter opts in only where its requests are idempotent and +its protocol exposes a cursor. Chunk planning and resumable partial state remain +OGC-specific capabilities rather than assumptions imposed on every service. Consequences ------------ diff --git a/docs/source/architecture/decisions/0006-api-neutral-transport.rst b/docs/source/architecture/decisions/0006-api-neutral-transport.rst new file mode 100644 index 00000000..01cd9487 --- /dev/null +++ b/docs/source/architecture/decisions/0006-api-neutral-transport.rst @@ -0,0 +1,69 @@ +ADR 0006: Use an API-neutral transport layer +============================================ + +Status +------ + +Accepted + +Context +------- + +Several service adapters need the same low-level capabilities: guarded HTTP +clients, cursor pagination, bounded retry, response aggregation, progress, and a +sync-over-async bridge. Locating those capabilities inside a protocol package +would make unrelated services depend on protocol-specific implementation +details. Duplicating them would allow authentication, timeout, retry, and +failure behavior to drift. + +Decision +-------- + +``dataretrieval.transport`` is the internal API-neutral execution layer. It owns: + +- synchronous and asynchronous HTTP client lifecycle and timeout defaults; +- host-scoped API-key construction and redirect-time credential stripping; +- callback-driven cursor pagination; +- bounded retry with exponential backoff, full jitter, and capped + ``Retry-After`` handling; +- DataFrame and HTTP-response aggregation; +- best-effort progress reporting; and +- the sync-over-async blocking-portal bridge. + +Transport depends only on stable package leaves and third-party infrastructure. +It must not import OGC modules or service adapters. Service adapters inject +request construction, response parsing, cursor extraction, and API-specific +error details. + +OGC retains its protocol concerns: dialects, CQL2, request construction, feature +shaping, URL-byte chunk planning, resumable ``ChunkedCall`` state, and typed +interruption handles. Thin imports at previous private OGC and utility paths +preserve compatibility while production consumers use the canonical transport +modules. + +Automatic retry is enabled only on active, idempotent request paths. Deprecated +NWIS calls retain their compatibility behavior. A failed pagination or fan-out +operation raises rather than returning successful siblings as an apparently +complete result. + +Consequences +------------ + +- Water Use has no dependency on OGC implementation modules. +- OGC and non-OGC adapters share authentication, timeout, retry, pagination, + aggregation, progress, and sync-dispatch policy where their semantics match. +- Service-specific request and result contracts remain explicit instead of + being forced into a universal adapter abstraction. +- Retry can increase latency and quota consumption, so attempt counts and waits + remain bounded and cancellation signals are never wrapped. +- The transport package is internal infrastructure, not a new public API + promise. + +Compliance +---------- + +``tests/architecture_test.py`` enforces transport dependency direction, an +acyclic transport graph, and Water Use isolation from OGC. Component and adapter +tests cover cursor termination, row caps, response aggregation, retry +exhaustion, ``Retry-After`` limits, cancellation, no-partial fan-out behavior, +credential host scoping, and compatibility imports. diff --git a/docs/source/architecture/decisions/0007-adapter-facades.rst b/docs/source/architecture/decisions/0007-adapter-facades.rst new file mode 100644 index 00000000..2244f09c --- /dev/null +++ b/docs/source/architecture/decisions/0007-adapter-facades.rst @@ -0,0 +1,64 @@ +ADR 0007: Organize service adapters behind stable facades +========================================================= + +Status +------ + +Accepted + +Context +------- + +A service facade can remain stable while its implementation grows for unrelated +upstream collections. Keeping every Water Data getter in one module coupled +changes to time series, monitoring metadata, field measurements, reference +catalogs, Samples, statistics, and generalized CQL queries. Active service +modules also relied on Python's implicit wildcard-export behavior, making their +intended public surfaces difficult to distinguish from imported helpers. + +Decision +-------- + +``dataretrieval.waterdata.api`` is a compatibility facade with no collection +logic. Implementation functions are grouped by collection family in +``time_series``, ``metadata``, ``measurements``, ``reference``, ``samples``, and +``cql``. Existing focused modules continue to own ratings, nearest-value +selection, Statistics API execution, shared Water Data policy, and type +vocabularies. + +The facade re-exports the established functions and preserves their signatures, +identity at ``dataretrieval.waterdata``, legacy ``__module__`` value, and private +Samples constants used by compatibility tests. Collection-family modules do not +import one another; shared behavior belongs in Water Data policy, OGC, or +transport modules. + +Active service and focused implementation modules declare explicit ``__all__`` +exports. Deprecated NWIS remains outside this modernization. Service adapters do +not import another adapter's implementation to obtain transport behavior. + +Return contracts remain service-specific. Tabular services generally return a +``(DataFrame, metadata)`` pair, while NLDI returns geospatial values directly, +StreamStats exposes response/domain objects, and ratings return parsed tables or +raw catalog features. Uniformity is not a reason to break these established +contracts. + +Consequences +------------ + +- Collection changes have a smaller implementation and test blast radius. +- Existing package and ``waterdata.api`` import paths remain stable. +- Explicit exports make accidental public-surface growth reviewable. +- More modules require a maintained facade and executable signature/export + snapshots. +- Tests are described as public-contract, adapter-contract, component, or + cross-component layers without forcing a disruptive move of established + files. + +Compliance +---------- + +``tests/contracts/public_api_test.py`` freezes Water Data imports, signatures, +facade identity, and compatibility names. ``tests/architecture_test.py`` +requires a logic-free facade, exact active-service exports, isolated collection +families, no lateral adapter reach-through, and separate OGC request +construction and schema execution. diff --git a/docs/source/architecture/decisions/index.rst b/docs/source/architecture/decisions/index.rst index f11aa4ba..2f893b8f 100644 --- a/docs/source/architecture/decisions/index.rst +++ b/docs/source/architecture/decisions/index.rst @@ -22,4 +22,6 @@ records sequentially. 0003-dependency-direction 0004-error-retry-resume 0005-legacy-nwis + 0006-api-neutral-transport + 0007-adapter-facades template diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index 906c07a4..c9d831ed 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -67,11 +67,13 @@ Public service facades ^^^^^^^^^^^^^^^^^^^^^^ ``dataretrieval.waterdata`` - Modern USGS Water Data API facade. Its generic adapter uses the four-symbol - OGC facade; internal Water Data modules import protocol helpers from their - canonical OGC modules. Water-Data-specific utilities own service policy and - wrappers without re-exporting private OGC helpers. Statistics, ratings, and - nearest-value operations live in separate modules. + Modern USGS Water Data API facade. ``waterdata.api`` is a logic-free + compatibility facade over collection-family modules: ``time_series``, + ``metadata``, ``measurements``, ``reference``, ``samples``, and ``cql``. + Focused modules own ratings, nearest-value selection, statistics execution, + shared service policy, and type vocabularies. Internal modules import + protocol helpers from their canonical OGC modules rather than re-exporting + them through Water Data utilities. ``dataretrieval.ngwmn`` NGWMN facade. Its only OGC dependency is the public OGC facade, which it @@ -79,14 +81,13 @@ Public service facades translation, and :class:`OgcDialect`. ``dataretrieval.wateruse`` - NWDC Water Use facade. Builds CSV requests and follows ``Link`` headers. - It currently reuses generic pagination and response-combining helpers from - private OGC modules; this is an explicitly recorded variance. + NWDC Water Use facade. Builds CSV requests, follows ``Link`` headers, and + uses API-neutral transport for bounded fan-out, retry, pagination, response + aggregation, and synchronous dispatch. It does not depend on OGC modules. ``dataretrieval.wqp``, ``dataretrieval.nldi``, and ``dataretrieval.streamstats`` - Service-specific adapters over the synchronous request infrastructure in - ``dataretrieval.utils``. Their return types intentionally reflect their - upstream data models. + Service-specific adapters over shared synchronous HTTP and bounded retry + policy. Their return types intentionally reflect their upstream data models. ``dataretrieval.nwis`` Deprecated legacy NWIS facade, scheduled for removal on or after @@ -100,31 +101,41 @@ Shared components (``__init__.py``) exposes the service-adapter seam: ``OgcDialect``, ``prepare_request_args``, ``get_ogc_data``, and ``fetch_ogc_request``. Internally, ``policy`` defines the dialect type and endpoint constants - (depends only on stdlib); ``requests`` owns request construction, argument - normalization, and queryables/schema lookup; ``engine`` orchestrates - pagination and sync-from-async; ``planning`` determines chunk boundaries; - ``chunking`` executes plans and retains resumable state; ``interruptions`` - defines the resumable failure contract; ``retry`` owns the bounded retry - policy; ``combining`` assembles results; and ``shaping``, ``dates``, - ``filters``, ``errors``, and ``progress`` isolate their named concerns. The - full runtime OGC graph, including the facade, is acyclic — - enforced by ``tests/architecture_test.py``. + (depends only on stdlib); ``context`` owns ambient base URL, dialect, and row + cap state; ``requests`` owns argument normalization and HTTP request + construction; ``schema`` executes queryables/schema requests; ``engine`` + supplies OGC cursor and response strategies to transport pagination; + ``planning`` determines chunk boundaries; ``chunking`` executes plans and + retains resumable state; ``interruptions`` defines the resumable failure + contract; ``retry`` classifies failures into OGC interruption types; and + ``shaping``, ``dates``, ``filters``, and ``errors`` isolate their named + protocol concerns. The full runtime OGC graph, including the facade, is + acyclic — enforced by ``tests/architecture_test.py``. + +``dataretrieval.transport`` + Internal API-neutral execution layer. Owns guarded client lifecycle and + timeouts, host-scoped authentication, cursor pagination, bounded retry, + response aggregation, progress, and sync-over-async dispatch. It imports no + service adapter or OGC protocol module, and it is not exposed as a public + framework API. ``dataretrieval.exceptions`` Stable error-policy leaf. It has no runtime third-party dependency and may be imported by every service without creating an infrastructure cycle. ``dataretrieval.utils`` - Shared metadata, data-shaping helpers, ambient context support, and the - legacy synchronous request path. Its broad responsibility is known debt; - new service-specific behavior should not be added there by default. + Shared metadata, data-shaping helpers, ambient context support, legacy + request composition, and compatibility imports for transport names that + historically lived here. New service-specific behavior should not be added + there by default. ``dataretrieval.codes`` and ``dataretrieval.rdb`` State/time-zone code conversion and RDB parsing leaves. The intended direction is:: - public facade -> service/protocol adapter -> shared policy/infrastructure + public facade -> service/protocol adapter -> API-neutral transport + -> stable policy/infrastructure -> third-party library / network Dependencies must not point from shared infrastructure back to a public service @@ -149,6 +160,34 @@ Underscore-prefixed symbols are implementation details even where existing internal adapters currently import them; those imports are known variances, not new extension points. +Service return contracts +------------------------ + +The library preserves meaningful upstream differences rather than forcing every +service into one return shape: + +- Water Data, NGWMN, and Water Use tabular getters return ``(DataFrame, + BaseMetadata)``. Geometry-bearing Water Data and NGWMN results may use a + ``GeoDataFrame`` in the first position when geopandas is installed. + ``BaseMetadata`` carries request URL, elapsed query time, response headers, + and comments where the upstream format provides them. +- WQP getters return ``(DataFrame, WQP_Metadata)``; the service-specific + metadata extends ``BaseMetadata`` with WQP query parameters and site lookup. +- ``waterdata.get_ratings`` returns a mapping of feature IDs to parsed rating + ``DataFrame`` objects by default, or the raw STAC feature list when downloads + are disabled. +- NLDI navigation functions return ``GeoDataFrame`` objects directly, or raw + GeoJSON-like dictionaries when ``as_json=True``; they do not add a metadata + tuple. +- StreamStats functions return raw ``httpx.Response`` objects or the + service-specific ``Watershed`` domain object, depending on the requested + format. +- Deprecated NWIS functions retain their established DataFrame and legacy + metadata contracts through the published deprecation window. + +Changing one of these shapes is a public compatibility change and requires the +project's deprecation process; consistency alone is not sufficient reason. + Interaction view ---------------- @@ -170,9 +209,10 @@ The retained ``ChunkedCall`` reissues only missing chunks and applies the same finalization path when resumed. Cancellation and non-transient programming errors take precedence over retry/resume wrapping. -Non-OGC services use simpler request paths where their protocols do not provide -the same paging or resume semantics. Later transport consolidation must preserve -those public contracts and must not invent unsupported upstream capabilities. +Non-OGC services use the same transport policy only where their protocols have +matching semantics. Retry and cursor pagination remain explicit adapter choices; +chunk planning and resumable interruptions remain OGC capabilities rather than +invented features of upstream APIs that do not provide them. Resource and configuration view ------------------------------- @@ -189,16 +229,18 @@ Resource and configuration view the execution throttle. ``API_USGS_RETRIES`` - Number of OGC retries after the first attempt; defaults to four. Backoff is - exponential with full jitter and honors bounded ``Retry-After`` values. + Number of retries after the first attempt on supported active request paths; + defaults to four. Backoff is exponential with full jitter and honors bounded + ``Retry-After`` values. Deprecated NWIS compatibility paths do not opt in. ``API_USGS_PROGRESS`` Controls best-effort progress display. Reporting failures must never change retrieval results. -HTTP timeouts and connection limits are centralized for existing paths. -``wateruse`` currently has its own smaller fan-out cap. These differences must -remain visible until a shared transport policy replaces them deliberately. +HTTP timeout, redirect, and authentication policy is centralized in +``dataretrieval.transport``. OGC subrequest fan-out and Water Use location +fan-out retain separate explicit concurrency caps because their upstream costs +and request shapes differ. Known architectural debt ------------------------ @@ -207,11 +249,9 @@ This view records categories and representative locations of debt. The fitness functions in ``tests/architecture_test.py`` are authoritative for exact current dependency allowlists. -- ``wateruse`` depends on private generic helpers located under ``ogc`` even - though NWDC is not an OGC service. -- ``waterdata/api.py`` and ``ogc/engine.py`` contain multiple reasons to change. -- Active non-OGC services do not yet share OGC's retry/resume capabilities. -- ``utils.py`` combines metadata, shaping, configuration, and transport duties. +- ``ogc/engine.py`` retains compatibility wrappers alongside OGC orchestration. +- ``utils.py`` combines metadata, shaping, ambient configuration, legacy + request composition, and transport compatibility imports. These are documented so guardrails distinguish accepted current dependencies from new erosion. They should be removed through small, test-protected changes, diff --git a/tests/architecture_test.py b/tests/architecture_test.py index 8ba2d864..35454790 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -18,19 +18,11 @@ "dataretrieval.wqp", ) -# These top-level modules currently reach into OGC. NGWMN is an OGC adapter -# that uses the small facade (``dataretrieval.ogc``) exclusively. Water Use's -# imports are an accepted temporary variance under ADR 0003. This allowlist is -# the authoritative exact inventory; the ADR owns the policy and rationale. -# Exact equality makes either growth or removal intentional. +# NGWMN is the only top-level OGC consumer and uses the small facade +# (``dataretrieval.ogc``) exclusively. Exact equality makes growth or removal +# an intentional architecture change. _ALLOWED_TOP_LEVEL_OGC_IMPORTS = { - "dataretrieval.ngwmn": { - "dataretrieval.ogc", - }, - "dataretrieval.wateruse": { - "dataretrieval.ogc.combining", - "dataretrieval.ogc.engine", - }, + "dataretrieval.ngwmn": {"dataretrieval.ogc"}, } _ENGINE_REQUEST_IMPORTS = { @@ -349,7 +341,7 @@ def test_default_header_calls_are_target_scoped() -> None: if isinstance(node.func, ast.Attribute) else None ) - if function_name != "_default_headers": + if function_name not in {"_default_headers", "default_headers"}: continue has_target = bool(node.args) or any( keyword.arg == "target_url" for keyword in node.keywords @@ -363,3 +355,275 @@ def test_default_header_calls_are_target_scoped() -> None: "_default_headers calls without destination URL context:\n" + "\n".join(violations) ) + + +# --- API-neutral transport boundaries --- + + +def test_transport_does_not_depend_on_ogc_or_services() -> None: + """Transport policy must point inward, never back to protocol adapters.""" + violations: list[str] = [] + transport_root = PACKAGE_ROOT / "transport" + for path in sorted(transport_root.rglob("*.py")): + module = _module_name(path) + for dependency in _runtime_imports(path): + if ( + dependency == "dataretrieval.ogc" + or dependency.startswith("dataretrieval.ogc.") + or dependency.startswith(_SERVICE_PREFIXES) + ): + violations.append(f"{module} -> {dependency}") + assert not violations, "Transport crossed an adapter boundary:\n" + "\n".join( + violations + ) + + +def test_wateruse_has_no_ogc_dependency() -> None: + """The non-OGC Water Use adapter must consume transport directly.""" + imports = _runtime_imports(PACKAGE_ROOT / "wateruse.py") + ogc_dependencies = { + dependency + for dependency in imports + if dependency == "dataretrieval.ogc" + or dependency.startswith("dataretrieval.ogc.") + } + assert not ogc_dependencies, ( + f"Water Use imported OGC implementation modules: {sorted(ogc_dependencies)}" + ) + + +def test_transport_runtime_graph_is_acyclic() -> None: + """The API-neutral transport package must remain a directed acyclic graph.""" + graph = { + module: { + dependency + for dependency in imports + if dependency == "dataretrieval.transport" + or dependency.startswith("dataretrieval.transport.") + } + for module, imports in _package_import_graph().items() + if module == "dataretrieval.transport" + or module.startswith("dataretrieval.transport.") + } + visiting: set[str] = set() + visited: set[str] = set() + + def visit(module: str, path: tuple[str, ...]) -> None: + if module in visiting: + start = path.index(module) + cycle = (*path[start:], module) + raise AssertionError( + f"Cycle in transport runtime graph: {' -> '.join(cycle)}" + ) + if module in visited: + return + visiting.add(module) + for dependency in graph.get(module, set()): + if dependency in graph: + visit(dependency, (*path, module)) + visiting.remove(module) + visited.add(module) + + for module in graph: + visit(module, ()) + + +# --- Adapter structure and public export boundaries --- + +_EXPECTED_MODULE_EXPORTS = { + "ngwmn.py": { + "get_sites", + "get_water_level", + "get_lithology", + "get_well_construction", + "get_providers", + }, + "nldi.py": { + "get_flowlines", + "get_basin", + "get_features", + "get_features_by_data_source", + "search", + }, + "streamstats.py": { + "download_workspace", + "get_sample_watershed", + "get_watershed", + "Watershed", + }, + "wateruse.py": { + "get_wateruse", + "WATERUSE_URL", + "MODELS", + "TIME_RESOLUTIONS", + "MAX_CONCURRENT_REQUESTS", + }, + "wqp.py": { + "get_results", + "what_sites", + "what_organizations", + "what_projects", + "what_activities", + "what_detection_limits", + "what_habitat_metrics", + "what_project_weights", + "what_activity_metrics", + "wqp_url", + "wqx3_url", + "WQP_Metadata", + }, + "waterdata/api.py": { + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_peaks", + "get_queryables", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", + }, + "waterdata/time_series.py": { + "get_daily", + "get_continuous", + "get_latest_continuous", + "get_latest_daily", + "get_stats_por", + "get_stats_date_range", + }, + "waterdata/metadata.py": { + "get_monitoring_locations", + "get_time_series_metadata", + "get_combined_metadata", + "get_field_measurements_metadata", + }, + "waterdata/measurements.py": { + "get_field_measurements", + "get_peaks", + "get_channel", + }, + "waterdata/reference.py": {"get_reference_table", "get_queryables"}, + "waterdata/samples.py": {"get_codes", "get_samples", "get_samples_summary"}, + "waterdata/cql.py": {"get_cql"}, + "waterdata/ratings.py": {"get_ratings"}, + "waterdata/nearest.py": {"get_nearest_continuous"}, + "waterdata/stats.py": {"get_data"}, + "waterdata/types.py": { + "CODE_SERVICES", + "METADATA_COLLECTIONS", + "SERVICES", + "WATERDATA_SERVICES", + "PROFILES", + "PROFILE_LOOKUP", + }, +} + + +def _literal_exports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "__all__" + for target in node.targets + ): + return set(ast.literal_eval(node.value)) + raise AssertionError(f"{path.relative_to(PACKAGE_ROOT.parent)} has no __all__") + + +def test_active_service_exports_are_explicit_and_stable() -> None: + for relative, expected in _EXPECTED_MODULE_EXPORTS.items(): + assert _literal_exports(PACKAGE_ROOT / relative) == expected, relative + + +def test_waterdata_api_is_a_logic_free_compatibility_facade() -> None: + path = PACKAGE_ROOT / "waterdata" / "api.py" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + definitions = [ + node.name + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + ] + assert not definitions, f"waterdata.api contains implementation: {definitions}" + + +def test_waterdata_collection_families_do_not_import_each_other() -> None: + families = { + "dataretrieval.waterdata.time_series", + "dataretrieval.waterdata.metadata", + "dataretrieval.waterdata.measurements", + "dataretrieval.waterdata.reference", + "dataretrieval.waterdata.samples", + "dataretrieval.waterdata.cql", + } + violations = [] + graph = _package_import_graph() + for module in families: + for dependency in graph[module]: + if dependency in families: + violations.append(f"{module} -> {dependency}") + assert not violations, "Lateral collection-family imports:\n" + "\n".join( + violations + ) + + +def test_service_adapters_do_not_reach_through_each_other() -> None: + adapters = { + "dataretrieval.ngwmn", + "dataretrieval.nldi", + "dataretrieval.streamstats", + "dataretrieval.waterdata", + "dataretrieval.wateruse", + "dataretrieval.wqp", + } + violations: list[str] = [] + for module, imports in _package_import_graph().items(): + owner = next( + ( + adapter + for adapter in adapters + if module == adapter or module.startswith(adapter + ".") + ), + None, + ) + if owner is None: + continue + for dependency in imports: + target = next( + ( + adapter + for adapter in adapters + if dependency == adapter or dependency.startswith(adapter + ".") + ), + None, + ) + if target is not None and target != owner: + violations.append(f"{module} -> {dependency}") + assert not violations, "Adapter-to-adapter imports:\n" + "\n".join( + sorted(set(violations)) + ) + + +def test_ogc_request_construction_does_not_execute_http() -> None: + path = PACKAGE_ROOT / "ogc" / "requests.py" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + transport_names = { + alias.name + for node in tree.body + if isinstance(node, ast.ImportFrom) + and node.module == "dataretrieval.transport.http" + for alias in node.names + } + assert transport_names == {"default_headers"} + assert "dataretrieval.ogc.schema" in _runtime_imports( + PACKAGE_ROOT / "ogc" / "shaping.py" + ) diff --git a/tests/contracts/README.md b/tests/contracts/README.md new file mode 100644 index 00000000..d05bed14 --- /dev/null +++ b/tests/contracts/README.md @@ -0,0 +1,19 @@ +# Test layers + +The suite uses four dependency-oriented layers without moving established tests: + +- **Public contract** (`tests/contracts/`): imports, exports, signatures, return + annotations, metadata/error promises, and compatibility paths. These tests use + public modules and no live services. +- **Adapter contract** (`waterdata_test.py`, `ngwmn_test.py`, `wateruse_test.py`, + `wqp_test.py`, `nldi_test.py`, `streamstats_test.py`): service request wiring, + response parsing, and documented protocol behavior. +- **Component** (`transport_test.py`, `waterdata_chunking_test.py`, + `waterdata_queryables_test.py`, `rdb_test.py`): one internal responsibility in + isolation. +- **Cross-component** (`architecture_test.py`, `headers_host_scoping_test.py`, + `waterdata_progress_test.py`): dependency fitness functions and behavior that + spans adapters, OGC, transport, or security boundaries. + +Live API cases remain in their existing adapter files and are not part of the +public-contract layer. diff --git a/tests/contracts/public_api_test.py b/tests/contracts/public_api_test.py new file mode 100644 index 00000000..d1d26d31 --- /dev/null +++ b/tests/contracts/public_api_test.py @@ -0,0 +1,309 @@ +"""Public import, export, and signature contracts for Water Data.""" +# ruff: noqa: E501 + +from __future__ import annotations + +import inspect + +from dataretrieval import waterdata +from dataretrieval.waterdata import api + +_EXPECTED_WATERDATA_ALL = [ + "CODE_SERVICES", + "FILTER_LANG", + "PROFILES", + "PROFILE_LOOKUP", + "SERVICES", + "WATERDATA_SERVICES", + "parallel_chunks", + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_nearest_continuous", + "get_peaks", + "get_queryables", + "get_ratings", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", +] + +_EXPECTED_API_NAMES = [ + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_peaks", + "get_queryables", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", +] + +_EXPECTED_SIGNATURES = { + "get_channel": "(monitoring_location_id: 'str | Iterable[str] | None' = None, field_visit_id: 'str | Iterable[str] | " + "None' = None, measurement_number: 'str | Iterable[str] | None' = None, time: 'str | Iterable[str] | " + "None' = None, channel_name: 'str | Iterable[str] | None' = None, channel_flow: 'str | Iterable[str] | " + "None' = None, channel_flow_unit: 'str | Iterable[str] | None' = None, channel_width: 'str | " + "Iterable[str] | None' = None, channel_width_unit: 'str | Iterable[str] | None' = None, channel_area: " + "'str | Iterable[str] | None' = None, channel_area_unit: 'str | Iterable[str] | None' = None, " + "channel_velocity: 'str | Iterable[str] | None' = None, channel_velocity_unit: 'str | Iterable[str] | " + "None' = None, channel_location_distance: 'str | Iterable[str] | None' = None, " + "channel_location_distance_unit: 'str | Iterable[str] | None' = None, channel_stability: 'str | " + "Iterable[str] | None' = None, channel_material: 'str | Iterable[str] | None' = None, " + "channel_evenness: 'str | Iterable[str] | None' = None, horizontal_velocity_description: 'str | " + "Iterable[str] | None' = None, vertical_velocity_description: 'str | Iterable[str] | None' = None, " + "longitudinal_velocity_description: 'str | Iterable[str] | None' = None, measurement_type: 'str | " + "Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = None, " + "channel_measurement_type: 'str | Iterable[str] | None' = None, properties: 'str | Iterable[str] | " + "None' = None, skip_geometry: 'bool | None' = None, bbox: 'list[float] | None' = None, limit: 'int | " + "None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: " + "'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_codes": "(code_service: 'CODE_SERVICES') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_combined_metadata": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, parameter_name: 'str | Iterable[str] | None' = None, " + "parameter_description: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | None' = None, data_type: " + "'str | Iterable[str] | None' = None, computation_identifier: 'str | Iterable[str] | None' = " + "None, thresholds: 'float | list[float] | None' = None, sublocation_identifier: 'str | " + "Iterable[str] | None' = None, primary: 'str | Iterable[str] | None' = None, " + "parent_time_series_id: 'str | Iterable[str] | None' = None, web_description: 'str | " + "Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = None, begin: " + "'str | Iterable[str] | None' = None, end: 'str | Iterable[str] | None' = None, agency_code: " + "'str | Iterable[str] | None' = None, agency_name: 'str | Iterable[str] | None' = None, " + "monitoring_location_number: 'str | Iterable[str] | None' = None, monitoring_location_name: " + "'str | Iterable[str] | None' = None, district_code: 'str | Iterable[str] | None' = None, " + "country_code: 'str | Iterable[str] | None' = None, country_name: 'str | Iterable[str] | " + "None' = None, state: 'str | Iterable[str] | None' = None, state_code: 'str | Iterable[str] " + "| None' = None, state_name: 'str | Iterable[str] | None' = None, county_code: 'str | " + "Iterable[str] | None' = None, county_name: 'str | Iterable[str] | None' = None, " + "minor_civil_division_code: 'str | Iterable[str] | None' = None, site_type_code: 'str | " + "Iterable[str] | None' = None, site_type: 'str | Iterable[str] | None' = None, " + "hydrologic_unit_code: 'str | Iterable[str] | None' = None, basin_code: 'str | Iterable[str] " + "| None' = None, altitude: 'str | Iterable[str] | None' = None, altitude_accuracy: 'str | " + "Iterable[str] | None' = None, altitude_method_code: 'str | Iterable[str] | None' = None, " + "altitude_method_name: 'str | Iterable[str] | None' = None, vertical_datum: 'str | " + "Iterable[str] | None' = None, vertical_datum_name: 'str | Iterable[str] | None' = None, " + "horizontal_positional_accuracy_code: 'str | Iterable[str] | None' = None, " + "horizontal_positional_accuracy: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_code: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_name: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum_name: 'str | Iterable[str] | None' = None, drainage_area: 'str | " + "Iterable[str] | None' = None, contributing_drainage_area: 'str | Iterable[str] | None' = " + "None, time_zone_abbreviation: 'str | Iterable[str] | None' = None, uses_daylight_savings: " + "'str | Iterable[str] | None' = None, construction_date: 'str | Iterable[str] | None' = " + "None, aquifer_code: 'str | Iterable[str] | None' = None, national_aquifer_code: 'str | " + "Iterable[str] | None' = None, aquifer_type_code: 'str | Iterable[str] | None' = None, " + "well_constructed_depth: 'str | Iterable[str] | None' = None, hole_constructed_depth: 'str | " + "Iterable[str] | None' = None, depth_source_code: 'str | Iterable[str] | None' = None, " + "properties: 'str | Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, bbox: " + "'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | None' = None, " + "filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: 'int | " + "None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_continuous": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] " + "| None' = None, statistic_id: 'str | Iterable[str] | None' = None, properties: 'str | " + "Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | None' = None, continuous_id: " + "'str | Iterable[str] | None' = None, approval_status: 'str | Iterable[str] | None' = None, " + "unit_of_measure: 'str | Iterable[str] | None' = None, qualifier: 'str | Iterable[str] | None' = " + "None, value: 'str | Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = " + "None, time: 'str | Iterable[str] | None' = None, limit: 'int | None' = None, filter: 'str | None' " + "= None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: 'int | " + "None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_cql": "(service: 'WATERDATA_SERVICES', cql: 'str | dict[str, Any]', *, properties: 'str | Iterable[str] | None' " + "= None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, skip_geometry: 'bool | None' = " + "None, convert_type: 'bool' = True) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_daily": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] | " + "None' = None, statistic_id: 'str | Iterable[str] | None' = None, properties: 'str | Iterable[str] | " + "None' = None, time_series_id: 'str | Iterable[str] | None' = None, daily_id: 'str | Iterable[str] | " + "None' = None, approval_status: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, qualifier: 'str | Iterable[str] | None' = None, value: 'str | " + "Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = None, skip_geometry: 'bool " + "| None' = None, time: 'str | Iterable[str] | None' = None, bbox: 'list[float] | None' = None, limit: " + "'int | None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, " + "convert_type: 'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> " + "'tuple[pd.DataFrame, BaseMetadata]'", + "get_field_measurements": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, observing_procedure_code: 'str | Iterable[str] | None' = " + "None, properties: 'str | Iterable[str] | None' = None, field_visit_id: 'str | " + "Iterable[str] | None' = None, approval_status: 'str | Iterable[str] | None' = None, " + "unit_of_measure: 'str | Iterable[str] | None' = None, qualifier: 'str | Iterable[str] | " + "None' = None, value: 'str | Iterable[str] | None' = None, last_modified: 'str | " + "Iterable[str] | None' = None, observing_procedure: 'str | Iterable[str] | None' = None, " + "vertical_datum: 'str | Iterable[str] | None' = None, measuring_agency: 'str | " + "Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, time: 'str | " + "Iterable[str] | None' = None, bbox: 'list[float] | None' = None, limit: 'int | None' = " + "None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: " + "'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_field_measurements_metadata": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: " + "'str | Iterable[str] | None' = None, parameter_name: 'str | Iterable[str] | None' " + "= None, parameter_description: 'str | Iterable[str] | None' = None, begin: 'str | " + "Iterable[str] | None' = None, end: 'str | Iterable[str] | None' = None, " + "last_modified: 'str | Iterable[str] | None' = None, properties: 'str | " + "Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, bbox: " + "'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | None' = " + "None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, " + "max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_latest_continuous": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | None' = None, " + "properties: 'str | Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | " + "None' = None, latest_continuous_id: 'str | Iterable[str] | None' = None, approval_status: " + "'str | Iterable[str] | None' = None, unit_of_measure: 'str | Iterable[str] | None' = None, " + "qualifier: 'str | Iterable[str] | None' = None, value: 'str | Iterable[str] | None' = None, " + "last_modified: 'str | Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, " + "time: 'str | Iterable[str] | None' = None, bbox: 'list[float] | None' = None, limit: 'int | " + "None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, " + "convert_type: 'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> " + "'tuple[pd.DataFrame, BaseMetadata]'", + "get_latest_daily": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | None' = None, properties: " + "'str | Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | None' = None, " + "latest_daily_id: 'str | Iterable[str] | None' = None, approval_status: 'str | Iterable[str] | " + "None' = None, unit_of_measure: 'str | Iterable[str] | None' = None, qualifier: 'str | " + "Iterable[str] | None' = None, value: 'str | Iterable[str] | None' = None, last_modified: 'str | " + "Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, time: 'str | Iterable[str] | " + "None' = None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | " + "None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: " + "'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_monitoring_locations": "(monitoring_location_id: 'str | Iterable[str] | None' = None, agency_code: 'str | " + "Iterable[str] | None' = None, agency_name: 'str | Iterable[str] | None' = None, " + "monitoring_location_number: 'str | Iterable[str] | None' = None, " + "monitoring_location_name: 'str | Iterable[str] | None' = None, district_code: 'str | " + "Iterable[str] | None' = None, country_code: 'str | Iterable[str] | None' = None, " + "country_name: 'str | Iterable[str] | None' = None, state: 'str | Iterable[str] | None' = " + "None, state_code: 'str | Iterable[str] | None' = None, state_name: 'str | Iterable[str] " + "| None' = None, county_code: 'str | Iterable[str] | None' = None, county_name: 'str | " + "Iterable[str] | None' = None, minor_civil_division_code: 'str | Iterable[str] | None' = " + "None, site_type_code: 'str | Iterable[str] | None' = None, site_type: 'str | " + "Iterable[str] | None' = None, hydrologic_unit_code: 'str | Iterable[str] | None' = None, " + "basin_code: 'str | Iterable[str] | None' = None, altitude: 'str | Iterable[str] | None' " + "= None, altitude_accuracy: 'str | Iterable[str] | None' = None, altitude_method_code: " + "'str | Iterable[str] | None' = None, altitude_method_name: 'str | Iterable[str] | None' " + "= None, vertical_datum: 'str | Iterable[str] | None' = None, vertical_datum_name: 'str | " + "Iterable[str] | None' = None, horizontal_positional_accuracy_code: 'str | Iterable[str] " + "| None' = None, horizontal_positional_accuracy: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_code: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_name: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum_name: 'str | Iterable[str] | None' = None, drainage_area: 'str " + "| Iterable[str] | None' = None, contributing_drainage_area: 'str | Iterable[str] | None' " + "= None, time_zone_abbreviation: 'str | Iterable[str] | None' = None, " + "uses_daylight_savings: 'str | Iterable[str] | None' = None, construction_date: 'str | " + "Iterable[str] | None' = None, aquifer_code: 'str | Iterable[str] | None' = None, " + "national_aquifer_code: 'str | Iterable[str] | None' = None, aquifer_type_code: 'str | " + "Iterable[str] | None' = None, well_constructed_depth: 'str | Iterable[str] | None' = " + "None, hole_constructed_depth: 'str | Iterable[str] | None' = None, depth_source_code: " + "'str | Iterable[str] | None' = None, properties: 'str | Iterable[str] | None' = None, " + "skip_geometry: 'bool | None' = None, bbox: 'list[float] | None' = None, limit: 'int | " + "None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, " + "convert_type: 'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> " + "'tuple[pd.DataFrame, BaseMetadata]'", + "get_peaks": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] | " + "None' = None, time_series_id: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, time: 'str | Iterable[str] | None' = None, last_modified: 'str | " + "Iterable[str] | None' = None, water_year: 'int | list[int] | None' = None, year: 'int | list[int] | " + "None' = None, month: 'int | list[int] | None' = None, day: 'int | list[int] | None' = None, peak_since: " + "'int | list[int] | None' = None, properties: 'str | Iterable[str] | None' = None, skip_geometry: 'bool " + "| None' = None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | None' = " + "None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: 'int | None' = " + "None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_queryables": "(collection: 'str') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_reference_table": "(collection: 'str', limit: 'int | None' = None, query: 'dict[str, Any] | None' = None, " + "max_rows: 'int | None' = None) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_samples": "(ssl_check: 'bool' = True, service: 'SERVICES' = 'results', profile: 'PROFILES' = 'fullphyschem', " + "activity_media_name: 'str | Iterable[str] | None' = None, activity_start_date_lower: 'str | None' = " + "None, activity_start_date_upper: 'str | None' = None, activity_type_code: 'str | Iterable[str] | " + "None' = None, characteristic_group: 'str | Iterable[str] | None' = None, characteristic: 'str | " + "Iterable[str] | None' = None, characteristic_user_supplied: 'str | Iterable[str] | None' = None, " + "bbox: 'list[float] | None' = None, country_code: 'str | Iterable[str] | None' = None, state_code: " + "'str | Iterable[str] | None' = None, county_code: 'str | Iterable[str] | None' = None, " + "site_type_code: 'str | Iterable[str] | None' = None, site_type_name: 'str | Iterable[str] | None' = " + "None, usgs_pcode: 'str | Iterable[str] | None' = None, hydrologic_unit: 'str | Iterable[str] | None' " + "= None, monitoring_location_id: 'str | Iterable[str] | None' = None, organization_id: 'str | " + "Iterable[str] | None' = None, point_location_latitude: 'float | None' = None, " + "point_location_longitude: 'float | None' = None, point_location_within_miles: 'float | None' = None, " + "project_id: 'str | Iterable[str] | None' = None, record_identifier_user_supplied: 'str | " + "Iterable[str] | None' = None) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_samples_summary": "(monitoring_location_id: 'str', ssl_check: 'bool' = True) -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_stats_date_range": "(approval_status: 'str | None' = None, computation_type: 'str | Iterable[str] | None' = " + "None, country_code: 'str | Iterable[str] | None' = None, state: 'str | Iterable[str] | None' " + "= None, state_code: 'str | Iterable[str] | None' = None, county_code: 'str | Iterable[str] | " + "None' = None, start_date: 'str | None' = None, end_date: 'str | None' = None, " + "monitoring_location_id: 'str | Iterable[str] | None' = None, page_size: 'int' = 1000, " + "parent_time_series_id: 'str | Iterable[str] | None' = None, site_type_code: 'str | " + "Iterable[str] | None' = None, site_type_name: 'str | Iterable[str] | None' = None, " + "parameter_code: 'str | Iterable[str] | None' = None, interval_type: 'str | Iterable[str] | " + "None' = None, expand_percentiles: 'bool' = True) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_stats_por": "(approval_status: 'str | None' = None, computation_type: 'str | Iterable[str] | None' = None, " + "country_code: 'str | Iterable[str] | None' = None, state: 'str | Iterable[str] | None' = None, " + "state_code: 'str | Iterable[str] | None' = None, county_code: 'str | Iterable[str] | None' = None, " + "start_date: 'str | None' = None, end_date: 'str | None' = None, monitoring_location_id: 'str | " + "Iterable[str] | None' = None, page_size: 'int' = 1000, parent_time_series_id: 'str | Iterable[str] " + "| None' = None, site_type_code: 'str | Iterable[str] | None' = None, site_type_name: 'str | " + "Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] | None' = None, normal_type: " + "'str | None' = None, expand_percentiles: 'bool' = True) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_time_series_metadata": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, parameter_name: 'str | Iterable[str] | None' = None, " + "properties: 'str | Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | " + "None' = None, hydrologic_unit_code: 'str | Iterable[str] | None' = None, state: 'str | " + "Iterable[str] | None' = None, state_name: 'str | Iterable[str] | None' = None, " + "last_modified: 'str | Iterable[str] | None' = None, begin: 'str | Iterable[str] | None' " + "= None, end: 'str | Iterable[str] | None' = None, begin_utc: 'str | Iterable[str] | " + "None' = None, end_utc: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, computation_period_identifier: 'str | Iterable[str] | " + "None' = None, computation_identifier: 'str | Iterable[str] | None' = None, thresholds: " + "'float | list[float] | None' = None, sublocation_identifier: 'str | Iterable[str] | " + "None' = None, primary: 'str | Iterable[str] | None' = None, parent_time_series_id: 'str " + "| Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | None' = None, " + "web_description: 'str | Iterable[str] | None' = None, skip_geometry: 'bool | None' = " + "None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | " + "None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, " + "max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", +} + + +def test_waterdata_exports_are_stable() -> None: + assert waterdata.__all__ == _EXPECTED_WATERDATA_ALL + assert api.__all__ == _EXPECTED_API_NAMES + assert all(hasattr(waterdata, name) for name in waterdata.__all__) + + +def test_api_facade_preserves_function_contracts() -> None: + for name, expected_signature in _EXPECTED_SIGNATURES.items(): + package_function = getattr(waterdata, name) + facade_function = getattr(api, name) + assert package_function is facade_function + assert str(inspect.signature(facade_function)) == expected_signature + assert facade_function.__module__ == "dataretrieval.waterdata.api" + + +def test_api_private_samples_compatibility_names_remain() -> None: + assert isinstance(api._SAMPLES_PARAM_TO_API, dict) + assert isinstance(api._SAMPLES_LEGACY_KWARGS, dict) + assert callable(api.get_ogc_data) diff --git a/tests/nldi_test.py b/tests/nldi_test.py index 9092bbf3..1b7a5957 100644 --- a/tests/nldi_test.py +++ b/tests/nldi_test.py @@ -1,3 +1,5 @@ +from unittest import mock + import pytest from geopandas import GeoDataFrame @@ -46,6 +48,17 @@ def mock_request_data_sources(httpx_mock): ) +def test_query_nldi_opts_into_retry(monkeypatch): + """NLDI explicitly enables shared retry while NWIS remains unchanged.""" + response = mock.Mock() + response.json.return_value = {} + query = mock.Mock(return_value=response) + monkeypatch.setattr(nldi, "_query_with_retry", query) + + assert nldi._query_nldi("https://example.test", {}) == {} + query.assert_called_once_with("https://example.test", payload={}) + + def mock_request(httpx_mock, request_url, file_path): with open(file_path) as text: httpx_mock.add_response( diff --git a/tests/streamstats_test.py b/tests/streamstats_test.py index ee528693..c51a3789 100644 --- a/tests/streamstats_test.py +++ b/tests/streamstats_test.py @@ -60,3 +60,19 @@ def test_get_watershed_shape_raises_not_implemented(httpx_mock): httpx_mock.add_response(text=json.dumps(_SAMPLE)) with pytest.raises(NotImplementedError): get_watershed("NY", -74.524, 43.939, format="shape") + + +def test_get_watershed_retries_transient_failure(httpx_mock, monkeypatch): + """StreamStats retries a bounded transient before returning normally.""" + import dataretrieval.transport.retry as retry + + httpx_mock.add_response(status_code=503) + httpx_mock.add_response(text=json.dumps(_SAMPLE)) + monkeypatch.setenv("API_USGS_RETRIES", "1") + monkeypatch.setattr(retry, "_RETRY_BASE_BACKOFF", 0.0) + monkeypatch.setattr(retry, "_RETRY_MAX_BACKOFF", 0.0) + + response = get_watershed("NY", -74.524, 43.939) + + assert response.status_code == 200 + assert len(httpx_mock.get_requests()) == 2 diff --git a/tests/transport_test.py b/tests/transport_test.py new file mode 100644 index 00000000..35e9782f --- /dev/null +++ b/tests/transport_test.py @@ -0,0 +1,159 @@ +"""Component tests for the internal API-neutral transport layer.""" + +from __future__ import annotations + +import asyncio +from unittest import mock + +import httpx +import pandas as pd +import pytest + +import dataretrieval.ogc.combining as ogc_combining +import dataretrieval.ogc.progress as ogc_progress +import dataretrieval.ogc.retry as ogc_retry +import dataretrieval.transport.combining as combining +import dataretrieval.transport.progress as progress +import dataretrieval.transport.retry as retry +from dataretrieval.exceptions import RateLimited, ServiceUnavailable +from dataretrieval.transport.pagination import paginate +from dataretrieval.transport.sync import run_sync +from dataretrieval.utils import _raise_for_status + + +def _response( + status: int = 200, *, url: str = "https://example.test/page" +) -> httpx.Response: + return httpx.Response(status, request=httpx.Request("GET", url)) + + +def test_paginate_follows_cursor_and_aggregates_response() -> None: + first = _response(url="https://example.test/page/1") + second = _response(url="https://example.test/page/2") + first.headers["x-ratelimit-remaining"] = "9" + second.headers["x-ratelimit-remaining"] = "8" + client = mock.AsyncMock(spec=httpx.AsyncClient) + client.send.return_value = first + client.get.return_value = second + + cursors = {str(first.url): "next", str(second.url): None} + + def parse(response: httpx.Response) -> tuple[pd.DataFrame, str | None]: + return pd.DataFrame({"value": [str(response.url)]}), cursors[str(response.url)] + + async def follow(cursor: str, session: httpx.AsyncClient) -> httpx.Response: + assert cursor == "next" + return await session.get("https://example.test/page/2") + + frame, response = asyncio.run( + paginate( + httpx.Request("GET", first.url), + parse_response=parse, + follow_up=follow, + raise_for_status=_raise_for_status, + client=client, + ) + ) + + assert frame["value"].tolist() == [str(first.url), str(second.url)] + assert response.url == first.url + assert response.headers["x-ratelimit-remaining"] == "8" + + +def test_paginate_stops_on_repeated_cursor_and_respects_row_cap() -> None: + first = _response(url="https://example.test/page/1") + second = _response(url="https://example.test/page/2") + client = mock.AsyncMock(spec=httpx.AsyncClient) + client.send.return_value = first + client.get.return_value = second + + def parse(response: httpx.Response) -> tuple[pd.DataFrame, str]: + return pd.DataFrame({"value": [1, 2]}), "same-cursor" + + async def follow(cursor: str, session: httpx.AsyncClient) -> httpx.Response: + return await session.get(str(second.url)) + + frame, _ = asyncio.run( + paginate( + httpx.Request("GET", first.url), + parse_response=parse, + follow_up=follow, + raise_for_status=_raise_for_status, + client=client, + row_cap=3, + ) + ) + + assert frame["value"].tolist() == [1, 2, 1] + assert client.get.await_count == 1 + + +def test_retry_sync_retries_transient_then_succeeds(monkeypatch) -> None: + attempts = 0 + slept: list[float] = [] + monkeypatch.setattr(retry.time, "sleep", slept.append) + + def operation() -> str: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ServiceUnavailable("temporary") + return "ok" + + result = retry.retry_sync( + operation, + retry.RetryPolicy(max_retries=1, base_backoff=0, max_backoff=0), + ) + + assert result == "ok" + assert attempts == 2 + assert slept == [0] + + +def test_retry_sync_honors_cap_and_does_not_catch_cancellation(monkeypatch) -> None: + sleep = mock.Mock() + monkeypatch.setattr(retry.time, "sleep", sleep) + policy = retry.RetryPolicy(max_retries=2, retry_after_cap=60) + + with pytest.raises(RateLimited): + retry.retry_sync( + lambda: (_ for _ in ()).throw(RateLimited("later", retry_after=61)), + policy, + ) + sleep.assert_not_called() + + with pytest.raises(KeyboardInterrupt): + retry.retry_sync( + lambda: (_ for _ in ()).throw(KeyboardInterrupt()), + policy, + ) + + +def test_shared_status_mapping_preserves_retry_after() -> None: + response = httpx.Response( + 429, + headers={"Retry-After": "2.5"}, + request=httpx.Request("GET", "https://example.test"), + ) + with pytest.raises(RateLimited) as exc_info: + _raise_for_status(response) + assert exc_info.value.retry_after == 2.5 + + +def test_sync_bridge_runs_async_operation() -> None: + async def operation() -> str: + return "ok" + + assert run_sync(operation, service="test", error_url="https://example.test") == "ok" + + +def test_compatibility_paths_reference_transport_implementations() -> None: + assert ogc_combining._merge_response is combining._merge_response + assert ogc_progress.ProgressReporter is progress.ProgressReporter + assert ogc_retry.RetryPolicy is retry.RetryPolicy + assert ogc_retry._retry is retry.retry_async + + +def test_parse_retry_after_accepts_http_date() -> None: + assert retry.parse_retry_after("Wed, 21 Oct 2015 07:28:00 GMT") == 0.0 + assert retry.parse_retry_after("not-a-date") is None diff --git a/tests/utils_test.py b/tests/utils_test.py index 30950294..2a743cb6 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -33,6 +33,23 @@ def test_header(self, httpx_mock): assert response.status_code == 200 # GET was successful assert "user-agent" in response.request.headers + def test_opt_in_retry_recovers_from_transient(self, httpx_mock, monkeypatch): + """Active adapters can opt into bounded retry without changing NWIS.""" + import dataretrieval.transport.retry as retry + + url = "https://example.invalid/x" + request_url = f"{url}?a=1" + httpx_mock.add_response(method="GET", url=request_url, status_code=503) + httpx_mock.add_response(method="GET", url=request_url, text="ok") + monkeypatch.setenv("API_USGS_RETRIES", "1") + monkeypatch.setattr(retry, "_RETRY_BASE_BACKOFF", 0.0) + monkeypatch.setattr(retry, "_RETRY_MAX_BACKOFF", 0.0) + + response = utils._query_with_retry(url, {"a": "1"}) + + assert response.text == "ok" + assert len(httpx_mock.get_requests()) == 2 + class Test_error_taxonomy: """The unified request-error hierarchy. @@ -386,3 +403,17 @@ def test_resolves_an_iterable_element_wise(self): # A bad element fails the whole call (fail-fast). with pytest.raises(ValueError, match="not a recognized US state"): to_state(["WI", "XX"]) + + +def test_retrying_get_maps_invalid_url(monkeypatch): + """Direct active-service GETs do not leak raw httpx InvalidURL errors.""" + import httpx + + monkeypatch.setattr( + utils, + "_get", + mock.Mock(side_effect=httpx.InvalidURL("invalid URL")), + ) + + with pytest.raises(exceptions.URLTooLong): + utils._get_with_retry("https://example.invalid") diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index ddefafcc..98d45ca3 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -40,7 +40,6 @@ ) from dataretrieval.ogc import chunking as _chunking from dataretrieval.ogc import engine as _engine -from dataretrieval.ogc import retry as _retry_mod from dataretrieval.ogc.chunking import ( ChunkedCall, _chunked_client, @@ -49,11 +48,6 @@ multi_value_chunked, parallel_chunks, ) -from dataretrieval.ogc.combining import ( - _QUOTA_HEADER, - _combine_chunk_frames, - _combine_chunk_responses, -) from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS from dataretrieval.ogc.interruptions import ( ChunkInterrupted, @@ -70,12 +64,20 @@ _safe_request_bytes, ) from dataretrieval.ogc.requests import _construct_api_requests -from dataretrieval.ogc.retry import ( +from dataretrieval.transport import retry as _retry_mod +from dataretrieval.transport.combining import ( + _QUOTA_HEADER, + _combine_chunk_frames, + _combine_chunk_responses, +) +from dataretrieval.transport.retry import ( _RETRIES_DEFAULT, RetryPolicy, - _retry, _retryable, ) +from dataretrieval.transport.retry import ( + retry_async as _retry, +) from dataretrieval.utils import HTTPX_DEFAULTS diff --git a/tests/waterdata_progress_test.py b/tests/waterdata_progress_test.py index dc752591..bda01fad 100644 --- a/tests/waterdata_progress_test.py +++ b/tests/waterdata_progress_test.py @@ -17,11 +17,11 @@ import pandas as pd import pytest -from dataretrieval.ogc import progress as _progress from dataretrieval.ogc.chunking import ChunkedCall from dataretrieval.ogc.engine import _paginate, _walk_pages from dataretrieval.ogc.planning import ChunkPlan -from dataretrieval.ogc.progress import ( +from dataretrieval.transport import progress as _progress +from dataretrieval.transport.progress import ( ProgressReporter, current, progress_context, diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index df317244..542d03ff 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -567,7 +567,7 @@ def test_get_daily_max_rows_is_excluded_from_request_and_forwarded(): # met, then truncate the combined frame to exactly N) is covered without a # network round-trip by the ``_row_cap`` / ``_finalize_ogc`` tests in # tests/waterdata_utils_test.py. - with mock.patch("dataretrieval.waterdata.api.get_ogc_data") as fake: + with mock.patch("dataretrieval.waterdata.time_series.get_ogc_data") as fake: fake.return_value = (pd.DataFrame(), mock.MagicMock(spec=[])) get_daily( monitoring_location_id="USGS-05427718", @@ -1086,7 +1086,7 @@ def test_get_daily_parameter_code_as_series(self): URL (or POST body). Post-fix, ``_normalize_str_iterable`` materializes it to ``list`` at the function boundary. """ - with mock.patch("dataretrieval.waterdata.api.get_ogc_data") as fake: + with mock.patch("dataretrieval.waterdata.time_series.get_ogc_data") as fake: fake.return_value = (pd.DataFrame(), mock.MagicMock(spec=[])) get_daily( monitoring_location_id="USGS-05427718", diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index c39a8b19..74841739 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -18,6 +18,7 @@ ServiceUnavailable, TransientError, ) +from dataretrieval.ogc.context import _row_cap from dataretrieval.ogc.dates import _format_api_dates from dataretrieval.ogc.engine import ( _next_req_url, @@ -28,7 +29,7 @@ _parse_retry_after, _raise_for_non_200, ) -from dataretrieval.ogc.requests import _check_ogc_requests, _row_cap +from dataretrieval.ogc.schema import _check_ogc_requests from dataretrieval.ogc.shaping import ( _arrange_cols, _get_resp_data, @@ -878,13 +879,10 @@ def test_parse_retry_after_clamps_negative_delta_to_zero(): assert _parse_retry_after("-0.5") == 0.0 -def test_parse_retry_after_returns_none_for_unparseable(): - """Garbage values (including the RFC 1123 HTTP-date form that the - HTTP spec allows but USGS doesn't actually send) surface as - ``None``, letting the chunker fall back to its own retry policy - instead of guessing a delay.""" +def test_parse_retry_after_supports_http_date_and_rejects_garbage(): + """Both standard header forms are accepted; malformed values use backoff.""" assert _parse_retry_after("not-a-date") is None - assert _parse_retry_after("Wed, 21 Oct 2099 07:28:00 GMT") is None + assert _parse_retry_after("Wed, 21 Oct 2099 07:28:00 GMT") > 0 def test_raise_for_non_200_raises_service_unavailable_for_5xx(): @@ -1057,7 +1055,7 @@ def test_ogc_getter_resolves_state_at_getter_layer(monkeypatch): """The OGC getters resolve the unified ``state`` into ``state_name`` themselves (any encoding), so the shared ``get_ogc_data`` wrapper stays state-agnostic.""" - import dataretrieval.waterdata.api as _api + import dataretrieval.waterdata.metadata as _metadata captured: dict = {} @@ -1065,8 +1063,8 @@ def fake_get_ogc_data(args, service, *a, **k): captured.update(args=args, service=service) return pd.DataFrame(), mock.Mock() - monkeypatch.setattr(_api, "get_ogc_data", fake_get_ogc_data) - _api.get_monitoring_locations(state="55") # FIPS in -> full name out + monkeypatch.setattr(_metadata, "get_ogc_data", fake_get_ogc_data) + _metadata.get_monitoring_locations(state="55") # FIPS in -> full name out assert captured["args"].get("state_name") == "Wisconsin" assert "state" not in captured["args"] diff --git a/tests/wateruse_test.py b/tests/wateruse_test.py index 00a843c8..0bfc6aa9 100644 --- a/tests/wateruse_test.py +++ b/tests/wateruse_test.py @@ -306,7 +306,27 @@ def test_fan_out_surfaces_final_rate_limit_header(httpx_mock): assert md.header["x-ratelimit-remaining"] == "850" -# (response aggregation now reuses ogc.combining._combine_chunk_responses; the +# (response aggregation uses transport.combining._combine_chunk_responses; the + + +def test_fan_out_failure_never_returns_partial_data(httpx_mock): + """A failed location aborts the call even when another location succeeded.""" + httpx_mock.add_response( + method="GET", + url=re.compile(r".*location=stateCd%3ARI.*"), + text=_CSV_P1, + ) + httpx_mock.add_response( + method="GET", + url=re.compile(r".*location=stateCd%3AWI.*"), + status_code=503, + json={"detail": "temporarily unavailable"}, + ) + + with pytest.raises(dataretrieval.ServiceUnavailable): + get_wateruse(model="wu-public-supply-wd", state=["RI", "WI"]) + + # integration test above pins the rate-limit-header behavior end-to-end.) @@ -414,3 +434,29 @@ def test_next_page_url_leaves_api_host_untouched(): def test_module_exposes_catalog_constants(): assert "wu-public-supply-wd" in wateruse.MODELS assert set(wateruse.TIME_RESOLUTIONS) == {"monthly", "annualcy", "annualwy"} + + +def test_initial_transient_is_retried(httpx_mock, monkeypatch): + """Water Use retries an initial transient without holding its semaphore.""" + import dataretrieval.transport.retry as retry + + url = re.compile(r".*location=stateCd%3ARI.*") + httpx_mock.add_response(method="GET", url=url, status_code=503) + httpx_mock.add_response(method="GET", url=url, text=_CSV_P1) + monkeypatch.setenv("API_USGS_RETRIES", "1") + monkeypatch.setattr(retry, "_RETRY_BASE_BACKOFF", 0.0) + monkeypatch.setattr(retry, "_RETRY_MAX_BACKOFF", 0.0) + + df, _ = get_wateruse(model="wu-public-supply-wd", state="RI") + + assert len(df) == 2 + assert len(httpx_mock.get_requests()) == 2 + + +def test_next_page_url_rejects_cross_host_link(): + response = httpx.Response( + 200, + headers={"link": '; rel="next"'}, + ) + with pytest.raises(RuntimeError, match="cross-host"): + _next_page_url(response) diff --git a/tests/wqp_test.py b/tests/wqp_test.py index e4d0dba0..e7f34e67 100644 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -4,6 +4,7 @@ import pytest from pandas import DataFrame +import dataretrieval.wqp as wqp from dataretrieval.wqp import ( WQP_Metadata, _check_kwargs, @@ -37,6 +38,23 @@ def _assert_wqp_metadata(md, request_url): assert md.comment is None +def test_get_results_opts_into_retry(monkeypatch): + """WQP explicitly enables retry at its shared query boundary.""" + response = mock.Mock( + text="ResultIdentifier,ResultMeasureValue\nA,1.0\n", + url="https://example.test", + elapsed=datetime.timedelta(), + headers={}, + ) + query = mock.Mock(return_value=response) + monkeypatch.setattr(wqp, "_query_with_retry", query) + + df, _ = wqp.get_results(legacy=True) + + assert len(df) == 1 + assert query.call_count == 1 + + def test_read_wqp_csv_preserves_leading_zero_codes(): """Regression: WQP code columns (HUCs, parameter codes, FIPS) carry significant leading zeros; a bare ``read_csv`` inferred them as int/float