diff --git a/NEWS.md b/NEWS.md index f4dbd033..59f58290 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +**09/22/2026:** `waterdata.get_continuous()` accepts `method_category`, a column the `continuous` collection added in September 2026: the RLMS method category code (`STNRD`, `LMTUS`, `EXPER` or `UNKWN`) for the method in effect over an observation's interval. It is returned on every record and is null for time series that have not been categorized. It could already be passed through `**queryables`; it is now a documented parameter. `get_latest_continuous()` is unchanged, because `latest-continuous` does not have the field. + +**09/22/2026:** The Water Data OGC getters now request **v1** of the Water Data APIs (`api.waterdata.usgs.gov/ogcapi/v1`), [released September 2026](https://waterdata.usgs.gov/blog/api-v1-release/); v0 stays online until June 2027. **New:** `WaterdataConfiguration(api_version="v0")`, or `api_version = "v0"` in the `[waterdata]` table of the configuration file, pins v0 during the transition. It changes only the version segment of the OGC path; Samples, Statistics and STAC are versioned separately and have no v1. **Behavior change:** `waterdata.get_time_series_metadata()` returns `begin` and `end` in UTC with a time zone, and no longer returns `begin_utc`, `end_utc`, `state_name` or `hydrologic_unit_code`. **Deprecation:** passing `begin_utc`, `end_utc`, `state`, `state_name` or `hydrologic_unit_code` to that getter, as a filter or in `properties`, emits a `DeprecationWarning` and sends the call to v0; this may be removed on or after 2027-06-01. Use `begin`, `end`, and `get_combined_metadata()` instead. **Behavior change:** `waterdata.get_field_measurements()` returns `time` as a date rather than a datetime, parsed to a tz-naive midnight timestamp as `get_daily()` already does; the time of day is in `time_of_day`. The `field-measurements-metadata` collection has no `time` field and is unaffected. + **09/09/2026:** **Bug fix:** code and identifier columns keep their leading zeros. A bare `pandas.read_csv` infers a zero-padded code as a number, so `waterdata.get_samples()` returned parameter code `00060` as `60` and HUC12 `070700050502` as `70700050502`, and `nwis.get_info()` returned `huc_cd` `02060005` as `2060005`. One rule now decides what a code column is — a name ending in `code`, the RDB abbreviation `_cd`, or a name containing `identifier`, `huc`, or `fips` — and every delimited response is parsed through it: the Samples and WQP CSV readers, `rdb.read_rdb` (which reads the names from the RDB header rather than the caller listing them), and the Water Use CSV pages. **Behavior change:** these columns now hold strings. `waterdata.get_samples()`: `USGSpcode`, `Location_HUCEightDigitCode`, `Location_HUCTwelveDigitCode`, `SampleCollectionMethod_Identifier` (`get_samples_summary()` shares the parse; no column in its current profile was affected). `nwis.get_info()`, `nwis.what_sites()`, and `nwis.get_record(service="site")`: `huc_cd`, `state_cd`, `county_cd`, `district_cd`. A comparison against a number — `df["USGSpcode"] == 60` — or a merge onto a numeric key now matches nothing instead of raising, so compare against the padded string (`== "00060"`) or call `.astype(int)` where the number is what you want. **Behavior change:** a count whose name reads as an identifier is numeric again. WQP's `AlternateLocation_IdentifierCount` has been read as text since 05/31/2026 because "Identifier" appears in its name; a name ending in `count` is now excluded from the rule, so the same column has one dtype in every service that reports it. Measurement columns are unchanged, and the `waterdata` OGC getters and `ngwmn` were never affected: their JSON responses deliver codes as strings and numeric coercion there is limited to a fixed list of measurement columns. **Correction to the 1.2.0 notes:** the same fix was applied to the nine `wqp` getters on 05/31/2026 and never recorded here — `wqp.get_results()` and the `what_*` getters have returned HUCs, parameter codes, and FIPS codes as strings since that release. **09/01/2026:** **Announcement:** We at USGS Water Data for the Nation want your feedback! Tell us how we're doing by taking our quick [survey](https://usgswaterresources.gov1.qualtrics.com/jfe/form/SV_07gX8G1DeOtVrH8), available through September 2026. diff --git a/dataretrieval/_configuration_core.py b/dataretrieval/_configuration_core.py index 62b3a6c4..e6041af0 100644 --- a/dataretrieval/_configuration_core.py +++ b/dataretrieval/_configuration_core.py @@ -11,6 +11,7 @@ import math import os +import re import stat import sys import warnings @@ -26,11 +27,16 @@ from dataretrieval.exceptions import ConfigurationError #: Settings only an adapter can hold, because they name one service. No -#: package-wide value could mean anything for them: there is no one base URL. +#: package-wide value could mean anything for them: there is no one base URL, +#: and a version is a segment of one service's paths. #: #: The package-wide roster is :data:`SETTINGS`, declared below the class it is #: derived from. -ADAPTER_ONLY_SETTINGS: tuple[str, ...] = ("base_url",) +ADAPTER_ONLY_SETTINGS: tuple[str, ...] = ("base_url", "api_version") + +#: The adapter-only settings the file also refuses, so only a ``configure()`` +#: block can supply them (ADR 0011). +BLOCK_ONLY_SETTINGS: tuple[str, ...] = ("base_url",) #: Environment variable backing a setting (precedence step 2). #: @@ -52,12 +58,9 @@ #: Variables the environment is *refused* for, by setting. Named rather than left out of #: :data:`ENV_VARS`, so a caller who exports ``API_USGS_BASE_URL`` gets an error instead -#: of an ignored variable. The file refuses the same key in the same words -#: (:func:`_accepted_keys`): a base URL set outside the code could redirect the library -#: to another host without a reader of the script seeing it (ADR 0011). -#: -#: Derived from :data:`ADAPTER_ONLY_SETTINGS` so the file and the environment -#: cannot drift apart on which settings are code-only. +#: of an ignored variable. Derived from :data:`ADAPTER_ONLY_SETTINGS`, because a +#: variable applies to every adapter and those settings name one. The file +#: refuses only :data:`BLOCK_ONLY_SETTINGS` (ADR 0011). _REFUSED_ENV_VARS: dict[str, str] = { name: f"API_USGS_{name.upper()}" for name in ADAPTER_ONLY_SETTINGS } @@ -393,9 +396,7 @@ def _provenance(self) -> str: # Plain mixins rather than ``BaseConfiguration`` subclasses: a group has no # adapter and cannot be passed to :func:`configure`, so keeping it off that # branch leaves one linear base for the behavior. Frozen because a dataclass -# may not mix frozen and non-frozen bases; fields collect in reverse MRO order, -# so an adapter composing all four reads ``retries, stall_timeout, base_url, -# concurrency, parallel_chunks``. +# may not mix frozen and non-frozen bases. Fields collect in reverse MRO order. @dataclass(frozen=True) @@ -413,6 +414,13 @@ class _Redirectable: base_url: str | None = _UNSET +@dataclass(frozen=True) +class _Versioned: + """An adapter whose service publishes its API under a version path segment.""" + + api_version: str | None = _UNSET + + @dataclass(frozen=True) class _Concurrent: """An adapter that issues more than one request per call.""" @@ -771,6 +779,7 @@ def _coerce_count(value: object, label: str, optional: str) -> str: #: so the wider check cannot change a TOML outcome.) _TYPES: dict[str, Callable[[object, str, str], str]] = { "api_key": _coerce_string, + "api_version": _coerce_string, "base_url": _coerce_string, "progress": _coerce_progress, "concurrency": _coerce_concurrency, @@ -899,6 +908,26 @@ def _parse_base_url(raw: str, label: str) -> str: return value +#: The one shape a version takes in every Water Data path: ``v`` and digits. +_API_VERSION_RE = re.compile(r"^v\d+$") + + +def _parse_api_version(raw: str, label: str) -> str: + """Parse an API version: the segment the service publishes it under, ``v1``. + + Checked as a shape, not against a list. This module cannot know which + versions a service has published, and a closed list here would refuse a + version the service already serves until a release of this package named it. + """ + value = raw.strip() + if not _API_VERSION_RE.match(value): + raise ConfigurationError( + f"{label} must be the version segment of the service's path, " + f"such as 'v1' (got {raw!r})." + ) + return value + + def _parse_progress(raw: str, label: str, *, strict: bool) -> bool: """Parse a progress toggle, optionally preserving legacy env truthiness.""" value = raw.strip().lower() @@ -931,6 +960,7 @@ def _parse_progress(raw: str, label: str, *, strict: bool) -> bool: "parallel_chunks": _parse_parallel_chunks, "stall_timeout": _parse_seconds, "base_url": _parse_base_url, + "api_version": _parse_api_version, } @@ -1048,11 +1078,13 @@ def _adapter_file_settings( where = f"[{adapter}]" # An adapter this process has not imported declares no vocabulary, so its - # table is checked against the package-wide settings alone: refusing a key - # for want of a schema would make the file's validity depend on which - # optional extras happened to be installed. + # table is checked against every setting this release has a grammar for: + # refusing a key for want of a schema would make the file's validity depend + # on which optional extras happened to be installed. accepted = settings_for(adapter) - validated = _scalars(table, path, where, SETTINGS if accepted is None else accepted) + validated = _scalars( + table, path, where, _ALL_SETTINGS if accepted is None else accepted + ) label = f"{path} {where}" result: Mapping[str, tuple[str, str]] = MappingProxyType( {name: (value, label) for name, value in validated.items()} @@ -1230,7 +1262,7 @@ def _accepted_keys( # and :func:`_named_profile` refuses a table inside a profile, so a # sub-table here is always a profile rather than deeper nesting. continue - if key in ADAPTER_ONLY_SETTINGS: + if key in BLOCK_ONLY_SETTINGS: # Rejected from the file wherever it appears. A file that # redirects a data-retrieval library to another host is a # supply-chain hazard; an in-code block keeps the redirect @@ -1240,16 +1272,7 @@ def _accepted_keys( "configure() block, never from a file." ) if key not in allowed: - if key in SETTINGS: - # A real setting, in a table that does not read it. Unlike an - # unrecognized name -- which may belong to a newer release -- - # this cannot become meaningful later, and ignoring it without an error - # would leave a caller believing they had tuned something. See - # ADR 0010. - raise ConfigurationError( - f"{path}: {key!r} at {where} is not a setting that table " - f"accepts. It accepts: {', '.join(sorted(allowed))}." - ) + _reject_known_setting(key, path, where, allowed) warnings.warn( f"{path}: unknown setting {key!r} at {where} (ignored). " f"Known settings: {', '.join(SETTINGS)}.", @@ -1261,6 +1284,31 @@ def _accepted_keys( return out +def _reject_known_setting( + key: str, path: Path, where: str, allowed: frozenset[str] | tuple[str, ...] +) -> None: + """Raise if *key* is a setting this release knows but that table cannot use. + + Returns for an unknown name, which the caller warns about instead: an + unknown name may belong to a newer release, but a known setting in the + wrong table will never take effect, and ignoring it would leave the caller + believing it had (ADR 0010). + """ + if where == _TOP_LEVEL and key in ADAPTER_ONLY_SETTINGS: + # The generic message below would list only top-level settings, none of + # which is the one to write. + raise ConfigurationError( + f"{path}: {key!r} at {where} names one service and has no " + "package-wide value; set it in the table of the adapter it belongs " + "to, such as [waterdata]." + ) + if key in _ALL_SETTINGS: + raise ConfigurationError( + f"{path}: {key!r} at {where} is not a setting that table " + f"accepts. It accepts: {', '.join(sorted(allowed))}." + ) + + def _checked_table( table: dict[str, Any], path: Path, diff --git a/dataretrieval/_deprecation.py b/dataretrieval/_deprecation.py index 4c8f1968..1a4e1070 100644 --- a/dataretrieval/_deprecation.py +++ b/dataretrieval/_deprecation.py @@ -19,6 +19,10 @@ "waterdata.get_cql(service=)": "2027-08-09", "wateruse": "2027-08-11", "ogc.interruptions": "2027-08-25", + # Set by the service, not by this package: v0 of the collection serves these + # filters until June 2027, and the shim cannot outlive the endpoint it sends + # to (https://waterdata.usgs.gov/blog/api-v1-release/). + "waterdata.get_time_series_metadata(v0 filters)": "2027-06-01", } diff --git a/dataretrieval/configuration.py b/dataretrieval/configuration.py index c5b4be00..b666657e 100644 --- a/dataretrieval/configuration.py +++ b/dataretrieval/configuration.py @@ -72,6 +72,7 @@ # isort: off from dataretrieval._configuration_core import ( ADAPTERS as ADAPTERS, + BLOCK_ONLY_SETTINGS as BLOCK_ONLY_SETTINGS, CONFIG_PATH_ENV as CONFIG_PATH_ENV, CONCURRENCY_UNBOUNDED as CONCURRENCY_UNBOUNDED, DEFAULT_CONCURRENCY as DEFAULT_CONCURRENCY, @@ -94,6 +95,7 @@ _Frame as _Frame, _named_profiles as _named_profiles, _NO_FILE as _NO_FILE, + _parse_api_version as _parse_api_version, _parse_base_url as _parse_base_url, _parse_concurrency as _parse_concurrency, _parse_parallel_chunks as _parse_parallel_chunks, @@ -112,6 +114,7 @@ _SettingValue as _SettingValue, _UNSET as _UNSET, _validated_raw as _validated_raw, + _Versioned as _Versioned, config_path as config_path, settings_for as settings_for, ) @@ -655,6 +658,42 @@ def base_url(*, adapter: str | None = None, default: str | None = None) -> str | return _parse_base_url(raw, label) +@overload +def api_version(*, adapter: str | None = ...) -> str | None: ... + + +@overload +def api_version(*, adapter: str | None = ..., default: str) -> str: ... + + +def api_version( + *, adapter: str | None = None, default: str | None = None +) -> str | None: + """An adapter's configured API version, falling back to *default*. + + Settable from code or from the adapter's table in the file. The + environment refuses it (:data:`_REFUSED_ENV_VARS`), as it refuses every + adapter-only setting: a variable is package-wide, and a version belongs to + one service. + + Like :func:`base_url`, it has no package-wide default. The adapter passes + its own, as in ``api_version(adapter="waterdata", default=OGC_API_VERSION)``, + so the version is declared in the module that builds the path. + + Parameters + ---------- + adapter : str, optional + Whose version to resolve. + default : str, optional + Returned when nothing configured a version. If omitted, ``None`` is + returned; :func:`show_configuration` relies on this. + """ + raw, label, _source = _resolve("api_version", adapter) + if raw is None: + return default + return _parse_api_version(raw, label) + + # --- resolution ---------------------------------------------------------- #: Which source of the chain supplied a resolution. Machine-readable so a @@ -729,16 +768,26 @@ def _check_env_not_refused(name: str) -> None: Refused before anything is consulted, not when the chain reaches the environment source. The file and the environment refuse ``base_url`` as one rule (ADR 0011), so a variable that cannot work is not outranked, with no error, by a block that happens - to work. + to work. ``api_version`` is refused from the environment only; the message + points to the file's adapter table as well as the block. """ refused = _REFUSED_ENV_VARS.get(name) - if refused is not None and refused in os.environ: - raise ConfigurationError( - f"{_env_label(refused)} is set, but {name!r} may only be set " - "in code, in a configure() block, never from the environment. Unset " - f"it and pass the value on the adapter's configuration, e.g. " - f"WaterdataConfiguration({name}=...)." - ) + if refused is None or refused not in os.environ: + return + block_only = name in BLOCK_ONLY_SETTINGS + fault = ( + "may only be set in code, in a configure() block, never from the environment" + if block_only + else "names one service and has no package-wide value, so the environment " + "cannot set it" + ) + # Suggest the file only for the settings it accepts. + or_the_file = "" if block_only else ", or in that adapter's table of the file" + raise ConfigurationError( + f"{_env_label(refused)} is set, but {name!r} {fault}. Unset it and pass " + f"the value on the adapter's configuration, e.g. " + f"WaterdataConfiguration({name}=...){or_the_file}." + ) def _resolve_from_block( @@ -837,6 +886,7 @@ def _display_progress(_adapter: str | None = None) -> str: "parallel_chunks": lambda adapter: str(parallel_chunks(adapter=adapter)), "stall_timeout": lambda adapter: f"{stall_timeout(adapter=adapter):g}s", "base_url": lambda adapter: base_url(adapter=adapter) or "", + "api_version": lambda adapter: api_version(adapter=adapter) or "", } if set(_DISPLAYS) != set(_ALL_SETTINGS): # pragma: no cover - guards a coding error diff --git a/dataretrieval/waterdata/configuration.py b/dataretrieval/waterdata/configuration.py index eb353570..4f61fb14 100644 --- a/dataretrieval/waterdata/configuration.py +++ b/dataretrieval/waterdata/configuration.py @@ -18,6 +18,7 @@ _Redirectable, _register, _Retrying, + _Versioned, ) __all__ = ["WaterdataConfiguration"] @@ -25,7 +26,7 @@ @dataclass(frozen=True) class WaterdataConfiguration( - _Chunked, _Concurrent, _Redirectable, _Retrying, BaseConfiguration + _Chunked, _Concurrent, _Redirectable, _Retrying, _Versioned, BaseConfiguration ): """Settings for Water Data calls alone. @@ -45,9 +46,18 @@ class WaterdataConfiguration( base_url : str, optional Root to send Water Data requests to, instead of the service's own. The package appends its own paths, so one value redirects all four families together -- - ``/ogcapi/v0``, ``/samples-data``, ``/statistics/v0`` and ``/stac/v0``. Code - only: the file and the environment refuse it. The API key is scoped to the host - that accepts it, so a redirected call sends no key. + ``/ogcapi/``, ``/samples-data``, ``/statistics/v0`` and + ``/stac/v0``. Code only: the file and the environment refuse it. The API key is + scoped to the host that accepts it, so a redirected call sends no key. + api_version : str, optional + Version of the Water Data API to request, as the segment of its path: + ``"v1"``, which this release is written against, or ``"v0"`` while the + service keeps it online (until June 2027). It replaces that one segment, + so the Samples, Statistics and STAC families -- versioned separately, with + no v1 -- are unaffected. Settable in code or in the ``[waterdata]`` table + of the file, never from the environment. A version the response shaping + was not written for returns that version's columns as the service sends + them. concurrency : int or str, optional Cap on simultaneous sub-requests, or ``"unbounded"``. parallel_chunks : int, optional @@ -56,11 +66,11 @@ class WaterdataConfiguration( """ # The settings this service reads, named by the groups they come from: - # every adapter's retry settings, a redirectable base, and -- because Water - # Data queries divide along a URL byte budget and are executed concurrently - # -- both fan-out settings. Each group declares the setting itself once, in - # :mod:`dataretrieval.configuration`, which also defines its grammar and - # its coercion. + # every adapter's retry settings, a redirectable base, a versioned API, and + # -- because Water Data queries divide along a URL byte budget and are + # executed concurrently -- both fan-out settings. Each group declares the + # setting itself once, in :mod:`dataretrieval.configuration`, which also + # defines its grammar and its coercion. adapter: ClassVar[str] = "waterdata" diff --git a/dataretrieval/waterdata/endpoints.py b/dataretrieval/waterdata/endpoints.py index 8c153b87..438ba4ad 100644 --- a/dataretrieval/waterdata/endpoints.py +++ b/dataretrieval/waterdata/endpoints.py @@ -11,10 +11,17 @@ from dataretrieval import configuration as _configuration from dataretrieval.credentials import WATERDATA_BASE_URL +#: The version of the Water Data OGC API this release is written against. +#: Responses are shaped for it; a caller who pins another version through +#: ``WaterdataConfiguration(api_version=)`` gets that version's columns as sent. +OGC_API_VERSION = "v1" + #: Canonical paths below the Water Data root. They are not endpoints on their #: own: callers obtain complete destinations through the request-time functions -#: below (ADR 0011). -_OGC_API_PATH = "/ogcapi/v0" +#: below (ADR 0011). Only the OGC family takes its version at request time; the +#: Statistics service and the STAC catalog publish their own versions and have +#: no v1 (checked 2026-09-22). +_OGC_API_PATH = "/ogcapi" _SAMPLES_PATH = "/samples-data" _STATISTICS_API_PATH = "/statistics/v0" _RATINGS_CATALOG_PATH = "/stac/v0" @@ -22,7 +29,7 @@ # Default-value compatibility for the documented ``waterdata.utils`` constants. # Production collection-family modules do not import these raw values. _DEFAULT_BASE_URL = WATERDATA_BASE_URL -_DEFAULT_OGC_API_URL = f"{_DEFAULT_BASE_URL}{_OGC_API_PATH}" +_DEFAULT_OGC_API_URL = f"{_DEFAULT_BASE_URL}{_OGC_API_PATH}/{OGC_API_VERSION}" _DEFAULT_SAMPLES_URL = f"{_DEFAULT_BASE_URL}{_SAMPLES_PATH}" @@ -32,9 +39,23 @@ def _endpoint(path: str) -> str: return f"{root}{path}" -def ogc_api_url() -> str: - """Return the OGC collections endpoint for the effective configuration.""" - return _endpoint(_OGC_API_PATH) +def ogc_api_url(api_version: str | None = None) -> str: + """Return the OGC collections endpoint for the effective configuration. + + Parameters + ---------- + api_version : str, optional + Version for this one request, used by a getter that must reach a + version other than the configured one. ``None`` (the default) resolves + the version through the configuration chain. A getter passes this + instead of entering a ``configure`` block, because a block set by the + library would override the caller's own setting (ADR 0011). + """ + if api_version is None: + api_version = _configuration.api_version( + adapter="waterdata", default=OGC_API_VERSION + ) + return _endpoint(f"{_OGC_API_PATH}/{api_version}") def samples_url() -> str: @@ -53,6 +74,7 @@ def ratings_catalog_url() -> str: __all__ = [ + "OGC_API_VERSION", "ogc_api_url", "ratings_catalog_url", "samples_url", diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py index 5060a267..2b978ab9 100644 --- a/dataretrieval/waterdata/measurements.py +++ b/dataretrieval/waterdata/measurements.py @@ -74,7 +74,7 @@ def get_field_measurements( field-measurements schema in the OpenAPI reference for the available columns (e.g. geometry, id, monitoring_location_id, parameter_code, value, unit_of_measure, approval_status, qualifier, last_modified): - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/field-measurements field_visit_id : string or iterable of strings, optional A universally unique identifier (UUID) for the field visit. Multiple measurements may be made during a single field visit. @@ -261,7 +261,7 @@ def get_peaks( The collection covers both stage (parameter ``"00065"``, ``ft``) and discharge (parameter ``"00060"``, ``ft^3/s``); a typical streamgage has a series for each. Reference docs: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/peaks + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/peaks Parameters ---------- diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py index eb26d98e..a3de7bda 100644 --- a/dataretrieval/waterdata/metadata.py +++ b/dataretrieval/waterdata/metadata.py @@ -13,6 +13,7 @@ import pandas as pd +from dataretrieval._deprecation import REMOVALS, warn_deprecated from dataretrieval.waterdata.utils import ( _get_args, _with_state, @@ -343,6 +344,54 @@ def get_monitoring_locations( return get_ogc_data(args, collection, max_rows=max_rows) +#: Filters that v1 of ``time-series-metadata`` dropped (it answers 400 or 500 +#: to them), mapped to what to use instead. ``state`` resolves to +#: ``state_name`` before the check, so that entry covers both. +#: +#: The renames do not use ``_accept_legacy_kwargs`` (ADR 0012) because the old +#: name may also appear in ``properties``, and translating it would rename a +#: column the caller asked for by name. +_V0_ONLY_FILTERS: dict[str, str] = { + "begin_utc": "'begin'", + "end_utc": "'end'", + "state_name": "get_combined_metadata(state=...)", + "hydrologic_unit_code": "get_combined_metadata(hydrologic_unit_code=...)", +} + +#: Read from REMOVALS so the date is edited in one place (ADR 0012). +_V0_FILTER_REMOVAL = REMOVALS["waterdata.get_time_series_metadata(v0 filters)"] + + +def _time_series_metadata( + args: dict[str, Any], *, max_rows: int | None, state_given: bool +) -> tuple[pd.DataFrame, BaseMetadata]: + """Send the query, to v0 if it names a filter that v1 dropped. + + A dropped name counts whether it is a filter or a column in ``properties``; + v1 refuses both. The version is set on the request, not through a + ``configure`` block, which would override the caller's own setting + (ADR 0011). + """ + collection = "time-series-metadata" + requested = set(args.get("properties", ())) + legacy = [n for n in _V0_ONLY_FILTERS if n in args or n in requested] + if not legacy: + return get_ogc_data(args, collection, max_rows=max_rows) + for name in legacy: + spelled = "state" if name == "state_name" and state_given else name + warn_deprecated( + f"The {spelled!r} argument of get_time_series_metadata", + replacement=_V0_ONLY_FILTERS[name], + removal=_V0_FILTER_REMOVAL, + detail=( + "v1 of the Water Data API has no such filter, so this call is " + "sent to v0." + ), + stacklevel=3, + ) + return get_ogc_data(args, collection, max_rows=max_rows, api_version="v0") + + def get_time_series_metadata( monitoring_location_id: str | Iterable[str] | None = None, parameter_code: str | Iterable[str] | None = None, @@ -360,6 +409,7 @@ def get_time_series_metadata( unit_of_measure: str | Iterable[str] | None = None, computation_period_identifier: str | Iterable[str] | None = None, computation_identifier: str | Iterable[str] | None = None, + statistics_begin: str | Iterable[str] | None = None, thresholds: float | list[float] | None = None, sublocation_identifier: str | Iterable[str] | None = None, primary: str | Iterable[str] | None = None, @@ -401,18 +451,22 @@ def get_time_series_metadata( A human-understandable name corresponding to parameter_code. properties : string or iterable of strings, optional The columns to return from the query. - Available options are: begin, begin_utc, computation_identifier, - computation_period_identifier, end, end_utc, geometry, - hydrologic_unit_code, id, last_modified, monitoring_location_id, - parameter_code, parameter_description, parameter_name, - parent_time_series_id, primary, state_name, statistic_id, - sublocation_identifier, thresholds, unit_of_measure, web_description + Available options are: begin, computation_identifier, + computation_period_identifier, data_gap_interval, end, geometry, id, + last_modified, monitoring_location_id, parameter_code, + parameter_description, parameter_name, parent_time_series_id, primary, + statistic_id, statistics_begin, sublocation_identifier, thresholds, + unit_of_measure, web_description statistic_id : string or iterable of strings, optional A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. hydrologic_unit_code : string or iterable of strings, optional + Deprecated: v1 of the Water Data API does not filter this collection by + hydrologic unit. A call that passes it is sent to v0, with a + ``DeprecationWarning``, until v0 is retired in June 2027. Use + :func:`get_combined_metadata` with ``hydrologic_unit_code`` instead. A unique hydrologic unit code (HUC) of two to eight digits, based on the four levels of classification in the hydrologic unit system. The United States is divided and sub-divided into successively smaller hydrologic @@ -421,12 +475,13 @@ def get_time_series_metadata( each other, from the smallest (cataloging units) to the largest (regions). state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"``). + Deprecated, as ``hydrologic_unit_code`` is: v1 does not filter this + collection by state. Use :func:`get_combined_metadata` with ``state`` + instead. Accepts a full name (``"Wisconsin"``), a two-letter postal + code (``"WI"``), or a two-digit ANSI/FIPS code (``"55"``). state_name : string or iterable of strings, optional - The name of the state or state equivalent in which the monitoring location - is located. + Deprecated; see ``state``. The name of the state or state equivalent in + which the monitoring location is located. last_modified : string, optional The last time a record was refreshed in our database. A refresh may happen due to regular operational processes and does not necessarily @@ -445,23 +500,14 @@ def get_time_series_metadata( for the last 36 hours begin : string or iterable of strings, optional - This field contains the same information as "begin_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - end : string or iterable of strings, optional - This field contains the same information as "end_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - begin_utc : string or iterable of strings, optional - The datetime of the earliest observation in the time series. Together - with end, this field represents the period of record of a time series. - Note that some time series may have large gaps in their collection - record. This field is currently in the local time of the monitoring - location. We intend to update this in version v0 to use UTC with a time - zone. You can query this field using date-times or intervals, adhering - to RFC 3339, or using ISO 8601 duration objects. Intervals may be - bounded or half-bounded (double-dots at start or end). Only features - that have a begin that intersects the value of datetime are selected. + The datetime of the earliest observation in the time series, in UTC + with a time zone. Together with end, this field represents the period + of record of a time series. Note that some time series may have large + gaps in their collection record. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a begin that intersects the + value of datetime are selected. Examples: * A date-time: "2018-02-12T23:20:50Z" @@ -471,19 +517,18 @@ def get_time_series_metadata( * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours - end_utc : string or iterable of strings, optional - The datetime of the most recent observation in the time series. Data returned by - this endpoint updates at most once per day, and potentially less frequently than - that, and as such there may be more recent observations within a time series - than the time series end value reflects. Together with begin, this field - represents the period of record of a time series. It is additionally used to - determine whether a time series is "active". We intend to update this in - version v0 to use UTC with a time zone. - You can query this field using date-times or intervals, - adhering to RFC 3339, or using ISO 8601 duration objects. Intervals - may be bounded or half-bounded (double-dots at start or end). Only - features that have an end that intersects the value of datetime are - selected. + end : string or iterable of strings, optional + The datetime of the most recent observation in the time series, in UTC + with a time zone. Data returned by this endpoint updates at most once + per day, and potentially less frequently than that, and as such there + may be more recent observations within a time series than the time + series end value reflects. Together with begin, this field represents + the period of record of a time series. It is additionally used to + determine whether a time series is "active". You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have an end that intersects the + value of datetime are selected. Examples: * A date-time: "2018-02-12T23:20:50Z" @@ -493,6 +538,14 @@ def get_time_series_metadata( * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours + begin_utc : string or iterable of strings, optional + Deprecated: in v1 of the Water Data API, ``begin`` holds this value. A + call that passes it is sent to v0, with a ``DeprecationWarning``, until + v0 is retired in June 2027. + end_utc : string or iterable of strings, optional + Deprecated: in v1 of the Water Data API, ``end`` holds this value. A + call that passes it is sent to v0, with a ``DeprecationWarning``, until + v0 is retired in June 2027. unit_of_measure : string or iterable of strings, optional A human-readable description of the units of measurement associated with an observation. @@ -501,6 +554,12 @@ def get_time_series_metadata( computation_identifier : string or iterable of strings, optional Indicates whether the data from this time series represent a specific statistical computation. + statistics_begin : string or iterable of strings, optional + The year from which statistics are computed for this time series. When + it is populated, WDFN ignores data before that year when computing + statistics; when it is empty, the full period of record is used. + Published by v1 only, so a call routed to v0 by one of the deprecated + filters above cannot use it. thresholds : number or list of numbers, optional Thresholds represent known numeric limits for a time series, for example the historic maximum value for a parameter or a level below which a @@ -583,15 +642,13 @@ def get_time_series_metadata( ... begin="1990-01-01/..", ... ) """ - collection = "time-series-metadata" - # Build argument dictionary, omitting None values (resolving the unified # `state` argument into the OGC `state_name` queryable). args = _get_args( _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} ) - return get_ogc_data(args, collection, max_rows=max_rows) + return _time_series_metadata(args, max_rows=max_rows, state_given=state is not None) def get_combined_metadata( @@ -674,7 +731,7 @@ def get_combined_metadata( of record, …) in a single query. See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/combined-metadata + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/combined-metadata All ~35 location-catalog kwargs are accepted (``agency_code``, ``state_name``, ``drainage_area``, ``aquifer_code``, …) but only @@ -883,7 +940,7 @@ def get_field_measurements_metadata( field-measurement parameters a site has, and over what date range. See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements-metadata + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/field-measurements-metadata Parameters ---------- diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index c83232f0..0f4d85ad 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -267,6 +267,7 @@ def get_continuous( approval_status: str | Iterable[str] | None = None, unit_of_measure: str | Iterable[str] | None = None, qualifier: str | Iterable[str] | None = None, + method_category: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, last_modified: str | Iterable[str] | None = None, time: str | Iterable[str] | None = None, @@ -316,7 +317,8 @@ def get_continuous( The columns to return from the query. Available options are: geometry, id, time_series_id, monitoring_location_id, parameter_code, statistic_id, time, value, - unit_of_measure, approval_status, qualifier, last_modified + unit_of_measure, approval_status, qualifier, method_category, + last_modified time_series_id : string or iterable of strings, optional A unique identifier representing a single time series, corresponding to the id field in the time-series-metadata endpoint. @@ -345,6 +347,12 @@ def get_continuous( qualifier : string or iterable of strings, optional Any qualifiers associated with an observation, for instance whether a sensor may have been impacted by ice or whether values were estimated. + method_category : string or iterable of strings, optional + The RLMS method category code for the method in effect over the + observation's interval: "STNRD" (standardized, with known uncertainty + and full QA/QC), "LMTUS" (limited use: a modified or externally + sourced method), "EXPER" (experimental), or "UNKWN" (uncategorized). + Null for time series that have not been categorized. value : string or iterable of strings, optional The value of the observation. Values are transmitted as strings in the JSON response format to preserve precision. @@ -947,7 +955,7 @@ def get_stats_por( site_type_code: string, optional Site type code query parameter. A list of valid site type codes is available at - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + https://api.waterdata.usgs.gov/ogcapi/v1/collections/site-types/items. Example: "GW" (Groundwater site) site_type_name: string, optional Site type name query parameter. @@ -1089,12 +1097,12 @@ def get_stats_date_range( site_type_code: string, optional Site type code query parameter. A list of valid site type codes is available at - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + https://api.waterdata.usgs.gov/ogcapi/v1/collections/site-types/items. Example: "GW" (Groundwater site) site_type_name: string, optional Site type name query parameter. A list of valid site type names is available at - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + https://api.waterdata.usgs.gov/ogcapi/v1/collections/site-types/items. Example: "Well" parameter_code : string or iterable of strings, optional A 5-digit code identifying the constituent measured and the units of diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index ede2e117..0e9270ac 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -197,6 +197,7 @@ def get_ogc_data( cql_body: str | None = None, *, spatial: bool = True, + api_version: str | None = None, ) -> tuple[pd.DataFrame, BaseMetadata]: """Water-Data wrapper over :func:`~dataretrieval.ogc.get_ogc_data`. @@ -226,6 +227,10 @@ def get_ogc_data( spatial : bool, optional Whether the collection includes feature geometry. Water Data's typed feature collections do; reference tables pass ``False``. + api_version : str, optional + API version for this one request. ``None`` (the default) resolves it + through the configuration chain. See + :func:`~dataretrieval.waterdata.endpoints.ogc_api_url`. Returns ------- @@ -245,7 +250,7 @@ def get_ogc_data( # The endpoint is resolved from the active ContextVar at request time; # the documented ``OGC_API_URL`` constant remains the default-value # compatibility path rather than a production request destination. - base_url=ogc_api_url(), + base_url=ogc_api_url(api_version), spatial=spatial, extra_id_cols=_EXTRA_ID_COLS, dialect=WATERDATA_DIALECT, diff --git a/demos/USGS_WaterData_DailyStatistics_Examples.ipynb b/demos/USGS_WaterData_DailyStatistics_Examples.ipynb index c4680eb2..c1aacc9d 100644 --- a/demos/USGS_WaterData_DailyStatistics_Examples.ipynb +++ b/demos/USGS_WaterData_DailyStatistics_Examples.ipynb @@ -389,7 +389,7 @@ "## Statistics API tips\n", "\n", "The statistics API does **not** follow the OGC standards used by the\n", - "`api.waterdata.usgs.gov/ogcapi/v0/` endpoints. A few things to keep in mind:\n", + "`api.waterdata.usgs.gov/ogcapi/v1/` endpoints. A few things to keep in mind:\n", "\n", "- **Higher rate limits.** At the time of writing the statistics API allows ~4000\n", " requests/hour per IP (per token if a token is supplied).\n", diff --git a/demos/USGS_WaterData_Introduction_Examples.ipynb b/demos/USGS_WaterData_Introduction_Examples.ipynb index 0e1254a0..51cc60bf 100644 --- a/demos/USGS_WaterData_Introduction_Examples.ipynb +++ b/demos/USGS_WaterData_Introduction_Examples.ipynb @@ -7,7 +7,7 @@ "source": [ "# Introduction to the USGS Water Data APIs\n", "\n", - "The [USGS Water Data APIs](https://api.waterdata.usgs.gov/ogcapi/v0/) are the\n", + "The [USGS Water Data APIs](https://api.waterdata.usgs.gov/ogcapi/v1/) are the\n", "modern, OGC-based replacement for the legacy NWIS web services. In Python they are\n", "exposed through the `dataretrieval.waterdata` module, which will gradually replace\n", "the older `dataretrieval.nwis` functions.\n", @@ -86,7 +86,7 @@ "\n", "Use the `properties` argument to choose which columns come back. The full set of\n", "available properties for a collection is published in that collection's schema,\n", - "e.g. ." + "e.g. ." ] }, { @@ -164,7 +164,7 @@ "\n", "`get_monitoring_locations` returns site metadata. To browse the service in a\n", "web browser, visit\n", - ".\n", + ".\n", "\n", "A simple request for one known USGS site:" ] @@ -291,7 +291,7 @@ "### Daily values\n", "\n", "`get_daily` returns daily values. Browse it at\n", - "." + "." ] }, { @@ -346,7 +346,7 @@ "### Continuous\n", "\n", "`get_continuous` returns instantaneous (sensor) values. Browse it at\n", - ".\n", + ".\n", "\n", "This service currently allows at most **3 years** of data per request; with no\n", "`time` argument it returns the latest year. Continuous data have no geometry\n", diff --git a/demos/WaterData_demo.ipynb b/demos/WaterData_demo.ipynb index ec6518fb..35b0cc20 100644 --- a/demos/WaterData_demo.ipynb +++ b/demos/WaterData_demo.ipynb @@ -616,7 +616,7 @@ "The USGS Water Data APIs belong to the Water Data for the Nation (WDFN) group of applications and tools. These products exist under the broader National Water Information System (NWIS) program. Check out the links below for more information on the USGS Water Data APIs and other ways to download or view USGS water data:\n", "* [Water Data APIs Home](https://api.waterdata.usgs.gov/)\n", "* [Get an API Key](https://api.waterdata.usgs.gov/signup/)\n", - "* [Water Data API OGC Endpoint Catalog](https://api.waterdata.usgs.gov/ogcapi/v0/collections?f=html)\n", + "* [Water Data API OGC Endpoint Catalog](https://api.waterdata.usgs.gov/ogcapi/v1/collections?f=html)\n", "* [Water Data Download Form](https://api.waterdata.usgs.gov/download)\n", "* [Water Data for the Nation Home](https://waterdata.usgs.gov/)\n", "* [Water Data for the Nation Feedback Form](https://waterdata.usgs.gov/questions-comments/?referrerUrl=https://api.waterdata.usgs.gov)\n", diff --git a/docs/source/architecture/decisions/0011-configuration-profiles.rst b/docs/source/architecture/decisions/0011-configuration-profiles.rst index 05ec2473..cd97e6ee 100644 --- a/docs/source/architecture/decisions/0011-configuration-profiles.rst +++ b/docs/source/architecture/decisions/0011-configuration-profiles.rst @@ -257,3 +257,10 @@ Notes - The setting-group clause was added after the original decision, consolidating under ADR 0000 a rule the configuration core was stating in prose. It does not change behavior. +- ``api_version`` was added on 2026-09-22 as a second adapter-only setting, + when the Water Data OGC collections moved to v1. Unlike ``base_url``, the + file accepts it: the code-only rule above exists because a base URL can + redirect requests to another host, and a version cannot. The file's refusal + is therefore keyed on a separate ``BLOCK_ONLY_SETTINGS`` list, while the + environment still refuses every adapter-only setting, because a variable is + package-wide. diff --git a/docs/source/userguide/configuration.rst b/docs/source/userguide/configuration.rst index d6a5d102..e0295406 100644 --- a/docs/source/userguide/configuration.rst +++ b/docs/source/userguide/configuration.rst @@ -134,6 +134,12 @@ Settings a ``configure`` block: a file that redirected the library to another host would be a supply-chain hazard. See :ref:`configuration-redirect`. + * - ``api_version`` + - the version the release targets + - *(none — code or file)* + - Which version of one service's API to request, as the segment of its + path (``"v1"``). Per adapter; Water Data reads it for its OGC + collections. See :ref:`configuration-api-version`. Where settings come from @@ -279,8 +285,8 @@ Adapter Configuration Ac ==================================== ====================================== ======================================== ``waterdata`` ``waterdata.WaterdataConfiguration`` ``concurrency``, ``parallel_chunks``, ``retries``, ``stall_timeout``, - ``base_url`` -``ngwmn`` ``ngwmn.NgwmnConfiguration`` the same five + ``base_url``, ``api_version`` +``ngwmn`` ``ngwmn.NgwmnConfiguration`` the same, without ``api_version`` ``nwdc`` ``nwdc.NwdcConfiguration`` ``concurrency``, ``retries``, ``stall_timeout``, ``base_url`` ``wqp``, ``nldi``, ``streamstats`` ``wqp.WqpConfiguration`` and so on ``retries``, ``stall_timeout``, @@ -547,6 +553,49 @@ host you gave a credential to. If the mirror needs its own credential, it needs its own mechanism. +.. _configuration-api-version: + +Pinning the Water Data API version +---------------------------------- + +The Water Data OGC getters request v1 of the Water Data APIs and shape +responses for it. ``api_version`` sends an adapter's requests to another +version, either within a ``configure`` block or, from the file, for every +script that reads it: + +.. code-block:: python + + with dataretrieval.configure(WaterdataConfiguration(api_version="v0")): + df, md = waterdata.get_time_series_metadata( + monitoring_location_id="USGS-05114000" + ) + +.. code-block:: toml + + [waterdata] + api_version = "v0" + +The value is the version segment of the service's path, ``"v1"`` or ``"v0"``, +and it replaces only that segment. The Samples database, the statistics service +and the STAC catalog are versioned separately and have no v1, so a pin leaves +them unchanged. v0 stays online until June 2027, after which the service +redirects every v0 request to v1. + +A pinned version returns that version's columns as the service sends them. +v0 of ``time-series-metadata`` still has ``begin_utc``, ``end_utc``, +``state_name`` and ``hydrologic_unit_code``, which v1 removed, and v0 of +``field-measurements`` returns ``time`` as a datetime, where v1 returns a date +(parsed to a tz-naive midnight timestamp). +When a call to ``get_time_series_metadata`` names one of those four filters, +the getter sends that one call to v0 and emits a ``DeprecationWarning``. +It does not change your configuration, +so every other getter still uses the version you set. + +Unlike ``base_url``, the file accepts ``api_version``, +because a version cannot send a request to another host. +The environment refuses it, as it refuses every per-adapter setting. + + .. _configuration-secret-store: Keeping a key out of your environment entirely diff --git a/tests/configuration_test.py b/tests/configuration_test.py index e4045b79..d99fac98 100644 --- a/tests/configuration_test.py +++ b/tests/configuration_test.py @@ -27,14 +27,15 @@ from dataretrieval.waterdata import WaterdataConfiguration from dataretrieval.wqp import WqpConfiguration -WATERDATA_URL = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" +_WATERDATA_HOST = "https://api.waterdata.usgs.gov" +WATERDATA_URL = f"{_WATERDATA_HOST}/ogcapi/v1/collections/daily/items" # Where the base-URL tests redirect to. A host the suite cannot connect to, so a # redirect that failed to apply shows up as an unmocked request rather than as # a real one. _MIRROR = "https://mirror.example/waterdata" _MIRROR_RE = re.compile(r"^https://mirror\.example/") -_WATERDATA_RE = re.compile(r"^https://api\.waterdata\.usgs\.gov/") +_WATERDATA_RE = re.compile(rf"^{re.escape(_WATERDATA_HOST)}/") # One committed page of the ``daily`` collection, shared with the Water Data suite. Real # response shape rather than a hand-made stub, so a redirect is exercised through the @@ -400,6 +401,7 @@ def _resolved_settings() -> dict[object, object]: adapter=adapter ) snapshot[(adapter, "base_url")] = configuration.base_url(adapter=adapter) + snapshot[(adapter, "api_version")] = configuration.api_version(adapter=adapter) return snapshot @@ -1431,13 +1433,104 @@ def test_a_code_base_url_redirects_every_water_data_endpoint_family(httpx_mock): ) requested = [str(request.url) for request in httpx_mock.get_requests()] - assert requested[0].startswith(f"{_MIRROR}/ogcapi/v0/collections/daily/items") + assert requested[0].startswith(f"{_MIRROR}/ogcapi/v1/collections/daily/items") assert requested[1].startswith(f"{_MIRROR}/samples-data/codeservice/states") assert requested[2].startswith(f"{_MIRROR}/statistics/v0/observationNormals") assert requested[3].startswith(f"{_MIRROR}/stac/v0/search") assert all(_WATERDATA_RE.match(url) is None for url in requested) +def test_api_version_applies_from_code_and_from_the_file(config_file): + """A version, unlike a base URL, may come from the file, because it cannot + send a request to another host.""" + config_file('[waterdata]\napi_version = "v0"\n') + assert configuration.api_version(adapter="waterdata") == "v0" + # Adapter-only: no other adapter reads it, and it has no package-wide value. + assert configuration.api_version(adapter="ngwmn") is None + + with dataretrieval.configure(WaterdataConfiguration(api_version="v1")): + assert configuration.api_version(adapter="waterdata") == "v1" + assert configuration.api_version(adapter="waterdata") == "v0" + + +def test_api_version_at_the_top_level_names_the_table_to_move_it_to(config_file): + """The error names the table to move the line to, not the top-level + settings.""" + config_file('api_version = "v0"\n') + with pytest.raises( + configuration.ConfigurationError, match=r"such as \[waterdata\]" + ): + configuration.api_version(adapter="waterdata") + + +def test_api_version_in_a_table_that_does_not_read_it_raises(config_file): + """A known setting in the wrong table is an error, not a warning.""" + config_file('[wqp]\napi_version = "v0"\n') + with pytest.raises( + configuration.ConfigurationError, match="not a setting that table accepts" + ): + configuration.retries(adapter="wqp") + + +def test_api_version_must_be_a_version_path_segment(): + """Checked as a shape, not against a list of known versions.""" + for bad in ("1", "V1", "v1.2", "latest"): + with pytest.raises(configuration.ConfigurationError, match="such as 'v1'"): + WaterdataConfiguration(api_version=bad) + assert WaterdataConfiguration(api_version="v0").api_version == "v0" + + +def test_api_version_is_refused_from_the_environment(monkeypatch): + """Refused, as every adapter-only setting is. The message also points to + the file, which accepts it.""" + monkeypatch.setenv("API_USGS_API_VERSION", "v0") + with pytest.raises(configuration.ConfigurationError, match="table of the file"): + configuration.api_version(adapter="waterdata") + + +def test_a_code_api_version_changes_the_ogc_path_alone(httpx_mock): + """The setting changes the OGC version segment only; statistics and STAC + are versioned separately.""" + httpx_mock.add_response(json=_DAILY_PAGE) + httpx_mock.add_response(json={"data": []}) + httpx_mock.add_response(json={"features": []}) + httpx_mock.add_response(json=_DAILY_PAGE) + + with dataretrieval.configure(WaterdataConfiguration(api_version="v0")): + waterdata.get_daily(monitoring_location_id="USGS-05427718") + waterdata.get_stats_por( + monitoring_location_id="USGS-05427718", + parameter_code="00060", + start_date="01-01", + end_date="01-01", + ) + waterdata.get_ratings( + monitoring_location_id="USGS-05427718", download_and_parse=False + ) + # Outside the block: the version this release is written against. + waterdata.get_daily(monitoring_location_id="USGS-05427718") + + requested = [str(request.url) for request in httpx_mock.get_requests()] + assert requested[0].startswith( + f"{_WATERDATA_HOST}/ogcapi/v0/collections/daily/items" + ) + assert requested[1].startswith(f"{_WATERDATA_HOST}/statistics/v0/") + assert requested[2].startswith(f"{_WATERDATA_HOST}/stac/v0/search") + assert requested[3].startswith( + f"{_WATERDATA_HOST}/ogcapi/v1/collections/daily/items" + ) + + +def test_show_configuration_reports_a_pinned_api_version(config_file): + """show_configuration() reports a pinned version.""" + config_file('[waterdata]\napi_version = "v0"\n') + out = io.StringIO() + dataretrieval.show_configuration(stream=out) + override_section = out.getvalue().split("adapter overrides", 1)[1] + assert "api_version" in override_section + assert "v0" in override_section + + def test_a_code_base_url_redirects_the_adapters_requests(httpx_mock): """The setting has to redirect real requests, not only resolve to a string. @@ -1458,7 +1551,7 @@ def test_a_code_base_url_redirects_the_adapters_requests(httpx_mock): waterdata.get_daily(monitoring_location_id="USGS-05427718") direct_url = str(httpx_mock.get_requests()[-1].url) - assert redirected_url.startswith(f"{_MIRROR}/ogcapi/v0/collections/daily/items") + assert redirected_url.startswith(f"{_MIRROR}/ogcapi/v1/collections/daily/items") assert direct_url.startswith(WATERDATA_URL) streamstats_mirror = "https://mirror.example/streamstats" diff --git a/tests/contracts/README.md b/tests/contracts/README.md index c985f9da..c8d24e1d 100644 --- a/tests/contracts/README.md +++ b/tests/contracts/README.md @@ -9,7 +9,8 @@ The suite uses four dependency-oriented layers without moving established tests: `wqp_test.py`, `nldi_test.py`, `streamstats_test.py`): service request construction, response parsing, and documented protocol behavior. - **Component** (`transport_test.py`, `waterdata_chunking_test.py`, - `waterdata_queryables_test.py`, `rdb_test.py`, `_csv_test.py`): one internal + `waterdata_queryables_test.py`, `waterdata_endpoints_test.py`, + `waterdata_properties_test.py`, `rdb_test.py`, `_csv_test.py`): one internal responsibility in isolation. - **Cross-component** (`architecture_test.py`, `headers_host_scoping_test.py`, `waterdata_progress_test.py`): dependency fitness functions and behavior that diff --git a/tests/data/waterdata_ogc_fixtures.json b/tests/data/waterdata_ogc_fixtures.json index 0820e877..f53a53f4 100644 --- a/tests/data/waterdata_ogc_fixtures.json +++ b/tests/data/waterdata_ogc_fixtures.json @@ -265,18 +265,19 @@ ], "type": "Point" }, - "id": "1f6dacef-9405-4e72-a755-6d3ff6121051", + "id": "c7ab17e0-14cb-4998-b532-95353f75e6ee", "properties": { "approval_status": "Approved", - "last_modified": "2025-08-28T11:10:36.529563+00:00", + "last_modified": "2026-01-08T21:21:09.233391+00:00", + "method_category": "UNKWN", "monitoring_location_id": "USGS-06904500", "parameter_code": "00065", "qualifier": null, "statistic_id": "00011", - "time": "2025-01-01T00:00:00+00:00", + "time": "2025-09-22T16:00:00+00:00", "time_series_id": "b36569a0067443ac9425e850a3ac7baa", "unit_of_measure": "ft", - "value": "1.76" + "value": "0.39" }, "type": "Feature" }, @@ -288,18 +289,19 @@ ], "type": "Point" }, - "id": "0be08516-f3a0-4a60-b3bf-41fba6b0a3ea", + "id": "f76707d5-2a10-4de0-99c3-65b63b4a92db", "properties": { "approval_status": "Approved", - "last_modified": "2025-08-28T11:10:36.529563+00:00", + "last_modified": "2026-01-08T21:21:09.233391+00:00", + "method_category": "UNKWN", "monitoring_location_id": "USGS-06904500", "parameter_code": "00065", "qualifier": null, "statistic_id": "00011", - "time": "2025-01-01T00:15:00+00:00", + "time": "2025-09-22T16:15:00+00:00", "time_series_id": "b36569a0067443ac9425e850a3ac7baa", "unit_of_measure": "ft", - "value": "1.76" + "value": "0.39" }, "type": "Feature" } @@ -938,34 +940,30 @@ { "geometry": { "coordinates": [ - -89.35249999999999, - 43.20888888888889 + -74.5836111111111, + 40.9177777777778 ], "type": "Point" }, - "id": "04585e9fb4ec467aafa86c7c0b3f1439", + "id": "000068b2a47b45708c656bcff5264f54", "properties": { - "begin": "2013-10-01T06:00:00.000001", - "begin_utc": "2013-10-01T11:00:00+00:00", - "computation_identifier": "Decumulated", - "computation_period_identifier": "Points", - "data_gap_interval": "PT1H12M", - "end": "2025-10-01T05:45:00.000001", - "end_utc": "2025-10-01T10:45:00+00:00", - "hydrologic_unit_code": "070900020504", - "id": "04585e9fb4ec467aafa86c7c0b3f1439", - "last_modified": "2026-06-05T17:16:13.005607", - "monitoring_location_id": "USGS-05427718", - "parameter_code": "00045", - "parameter_description": "Precipitation, total, inches", - "parameter_name": "Precipitation", + "begin": "1983-11-03T05:00:00+00:00", + "computation_identifier": "Min", + "computation_period_identifier": "Daily", + "data_gap_interval": null, + "end": "1985-04-16T05:00:00+00:00", + "id": "000068b2a47b45708c656bcff5264f54", + "last_modified": "2017-05-07T20:53:29.551297", + "monitoring_location_id": "USGS-01379790", + "parameter_code": "00010", + "parameter_description": "Temperature, water, degrees Celsius", + "parameter_name": "Temperature, water", "parent_time_series_id": null, "primary": "Primary", - "state_name": "Wisconsin", - "statistic_id": null, + "statistic_id": "00002", "sublocation_identifier": null, "thresholds": [], - "unit_of_measure": "in", + "unit_of_measure": "degC", "web_description": null }, "type": "Feature" @@ -973,34 +971,30 @@ { "geometry": { "coordinates": [ - -89.35249999999999, - 43.20888888888889 + -112.13859444444445, + 40.780852777777774 ], "type": "Point" }, - "id": "0aa968f7d6494e2eb214d1829754532d", + "id": "00009b3da7e44a96a283bc413ff2de48", "properties": { - "begin": "1990-10-01T00:00:00.000001", - "begin_utc": "1990-10-01T05:00:00+00:00", - "computation_identifier": "Mean", - "computation_period_identifier": "Daily", - "data_gap_interval": null, - "end": "2025-09-30T00:00:00.000001", - "end_utc": "2025-09-30T05:00:00+00:00", - "hydrologic_unit_code": "070900020504", - "id": "0aa968f7d6494e2eb214d1829754532d", - "last_modified": "2025-12-19T19:35:48.365292", - "monitoring_location_id": "USGS-05427718", - "parameter_code": "91060", - "parameter_description": "Orthophosphate, water, filtered, pounds per day", - "parameter_name": "Orthophosphate, diss", - "parent_time_series_id": null, + "begin": "2006-05-18T12:00:00+00:00", + "computation_identifier": "Instantaneous", + "computation_period_identifier": "Points", + "data_gap_interval": "PT1H12M", + "end": "2026-09-22T16:00:00+00:00", + "id": "00009b3da7e44a96a283bc413ff2de48", + "last_modified": "2026-09-22T10:10:05.077878", + "monitoring_location_id": "USGS-10172640", + "parameter_code": "00060", + "parameter_description": "Discharge, cubic feet per second", + "parameter_name": "Discharge", + "parent_time_series_id": "769d7201a8044b4ab5218bac4cd282e8", "primary": "Primary", - "state_name": "Wisconsin", - "statistic_id": "00003", + "statistic_id": "00011", "sublocation_identifier": null, "thresholds": [], - "unit_of_measure": "lbs/day", + "unit_of_measure": "ft^3/s", "web_description": null }, "type": "Feature" diff --git a/tests/data/waterdata_queryables.json b/tests/data/waterdata_queryables.json index a9d9862c..93bf9411 100644 --- a/tests/data/waterdata_queryables.json +++ b/tests/data/waterdata_queryables.json @@ -119,6 +119,7 @@ "hydrologic_unit_code", "id", "last_modified", + "method_category", "minor_civil_division_code", "monitoring_location_id", "monitoring_location_name", @@ -487,14 +488,11 @@ ], "time-series-metadata": [ "begin", - "begin_utc", "computation_identifier", "computation_period_identifier", "data_gap_interval", "end", - "end_utc", "geometry", - "hydrologic_unit_code", "id", "last_modified", "monitoring_location_id", @@ -503,8 +501,8 @@ "parameter_name", "parent_time_series_id", "primary", - "state_name", "statistic_id", + "statistics_begin", "sublocation_identifier", "thresholds", "unit_of_measure", diff --git a/tests/headers_host_scoping_test.py b/tests/headers_host_scoping_test.py index 515435b3..f5aeeb03 100644 --- a/tests/headers_host_scoping_test.py +++ b/tests/headers_host_scoping_test.py @@ -27,7 +27,7 @@ def _api_token(self, monkeypatch: pytest.MonkeyPatch) -> None: def test_key_included_for_waterdata_host(self): """The key is added when target URL matches api.waterdata.usgs.gov.""" - url = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" + url = "https://api.waterdata.usgs.gov/ogcapi/v1/collections/daily/items" headers = _default_headers(url) assert headers.get("X-Api-Key") == self.FAKE_TOKEN @@ -51,7 +51,7 @@ def test_key_excluded_for_rating_asset_host(self): def test_key_excluded_for_lookalike_host(self): """The key is not sent to a typosquatting/lookalike domain.""" - url = "https://api.waterdata.usgs.gov.evil.com/ogcapi/v0/daily/items" + url = "https://api.waterdata.usgs.gov.evil.com/ogcapi/v1/daily/items" headers = _default_headers(url) assert "X-Api-Key" not in headers @@ -65,7 +65,7 @@ def test_key_excluded_when_no_token_set( ) -> None: """No key header at all when API_USGS_PAT is not set.""" monkeypatch.delenv("API_USGS_PAT") - headers = _default_headers("https://api.waterdata.usgs.gov/ogcapi/v0/daily") + headers = _default_headers("https://api.waterdata.usgs.gov/ogcapi/v1/daily") assert "X-Api-Key" not in headers def test_non_auth_headers_always_present(self): @@ -86,7 +86,7 @@ def test_key_excluded_over_cleartext_on_the_authorized_host(self): because of a hostname an attacker chose to keep -- reachable via a redirect or a server-supplied ``http://`` next-page link. """ - headers = _default_headers("http://api.waterdata.usgs.gov/ogcapi/v0/daily") + headers = _default_headers("http://api.waterdata.usgs.gov/ogcapi/v1/daily") assert "X-Api-Key" not in headers def test_sync_transport_withholds_key_on_downgrade_to_cleartext(self): diff --git a/tests/waterdata_endpoints_test.py b/tests/waterdata_endpoints_test.py new file mode 100644 index 00000000..1f4a1b80 --- /dev/null +++ b/tests/waterdata_endpoints_test.py @@ -0,0 +1,136 @@ +"""Live monitors for the API version each Water Data family serves. + +``waterdata/endpoints.py`` puts a version in the OGC, STAC and statistics +paths. An old version keeps answering after USGS publishes a new one, so no +other test fails when that happens. (Samples and NGWMN have no version segment.) + +Two conditions are checked separately: + +- **A new version is available**: the family's default no longer matches + :data:`_DEFAULT_VERSIONS`. +- **The version the package requests stopped working**: it no longer returns + data. + +The OGC and STAC roots publish a ``self`` link naming their default version, +so the version is read from it. OGC cannot be probed, because it answers 200 +with an empty body for any version segment. Statistics has no root document, +so it is probed; a missing statistics version answers 404. +""" + +import re + +import httpx +import pytest + +from dataretrieval.waterdata import endpoints + +#: The version each family serves by default, read from the live service on +#: 2026-09-22. This records what the service serves, not what the package +#: requests. Update it only after the package has moved to the new version, so +#: the test keeps failing until then. +_DEFAULT_VERSIONS = { + "ogcapi": "v1", + "stac": "v0", +} + +#: The endpoint function that builds each family's URL, so the version checked +#: is the one the package actually requests. +_FAMILIES = { + "ogcapi": endpoints.ogc_api_url, + "stac": endpoints.ratings_catalog_url, + "statistics": endpoints.statistics_api_url, +} + +#: A version segment anywhere in a path: ``/v0``, ``/v12/``, ``/v1?f=json``. +_VERSION_RE = re.compile(r"/(v\d+)(?=[/?#]|$)") + +#: Fail a hung request before the scheduled job's own timeout does. +_TIMEOUT = 60 + + +def _split_version(url: str) -> tuple[str, str]: + """Split *url* into its unversioned root and its version segment.""" + match = _VERSION_RE.search(url) + assert match is not None, f"no version segment in {url!r}" + return url[: match.start()] + "/", match.group(1) + + +def _served_version(root: str) -> str: + """The version in *root*'s ``self`` link, e.g. ``.../ogcapi/v1?f=json``.""" + response = httpx.get(root, timeout=_TIMEOUT, follow_redirects=True) + response.raise_for_status() + links = response.json().get("links") or [] + self_links = [link["href"] for link in links if link.get("rel") == "self"] + assert self_links, f"{root} published no self link: {links}" + return _split_version(self_links[0])[1] + + +@pytest.mark.live +@pytest.mark.parametrize("family", sorted(_DEFAULT_VERSIONS)) +def test_service_still_serves_the_recorded_default_version(family): + """The version a family serves by default is the one recorded here. + + On failure, move the pin in ``waterdata/endpoints.py`` to the new version + (check its release notes for dropped or renamed fields), then update + ``_DEFAULT_VERSIONS``. + """ + root, _ = _split_version(_FAMILIES[family]()) + served = _served_version(root) + + assert served == _DEFAULT_VERSIONS[family], ( + f"the {family} API now serves {served} by default, not " + f"{_DEFAULT_VERSIONS[family]}. Move the package to {served} and update " + "_DEFAULT_VERSIONS." + ) + + +@pytest.mark.live +def test_statistics_has_published_no_version_beyond_the_one_we_request(): + """Statistics has no root document, so the next version is probed. + + Both the unversioned root and ``/statistics/vN`` answer 404, so the + ``/docs`` page is what shows whether a version exists. + """ + url = _FAMILIES["statistics"]() + root, current = _split_version(url) + following = f"v{int(current.removeprefix('v')) + 1}" + + assert httpx.get(f"{url}/docs", timeout=_TIMEOUT).status_code == 200, ( + f"the statistics service stopped serving {current}, which this package " + "requests; check what replaced it." + ) + + probe = httpx.get(f"{root}{following}/docs", timeout=_TIMEOUT) + assert probe.status_code == 404, ( + f"the statistics service now answers for {following} " + f"(HTTP {probe.status_code}); check whether the package should move to " + "it, and whether it publishes a root document that would let this be " + "discovered rather than probed." + ) + + +@pytest.mark.live +@pytest.mark.parametrize("family", sorted(_FAMILIES)) +def test_the_version_this_package_requests_still_returns_data(family): + """The version the package pins still returns data. + + Checked on content, not status: OGC answers 200 with an empty body for a + version that does not exist. + """ + url = _FAMILIES[family]() + + if family == "statistics": + # No collections endpoint; its docs page is what proves it is up. + response = httpx.get(f"{url}/docs", timeout=_TIMEOUT) + response.raise_for_status() + assert response.text.strip(), f"{url}/docs returned an empty body" + return + + response = httpx.get(f"{url}/collections", timeout=_TIMEOUT) + response.raise_for_status() + collections = response.json().get("collections") + assert collections, ( + f"{url}/collections returned no collections, so the " + f"{_split_version(url)[1]} {family} API this package requests has stopped " + "serving. Move the package to the version the service now offers." + ) diff --git a/tests/waterdata_progress_test.py b/tests/waterdata_progress_test.py index 2286c81a..c509d320 100644 --- a/tests/waterdata_progress_test.py +++ b/tests/waterdata_progress_test.py @@ -44,7 +44,7 @@ def _run_walk_pages(*, geopd, req, client): # The Water Data host is the only one that accepts ``API_USGS_PAT``, and so the # only one where pointing the user at API-key registration is useful advice. -_KEYED_URL = "https://api.waterdata.usgs.gov/ogcapi/v0/" +_KEYED_URL = "https://api.waterdata.usgs.gov/ogcapi/v1/" @pytest.fixture(autouse=True) diff --git a/tests/waterdata_properties_test.py b/tests/waterdata_properties_test.py new file mode 100644 index 00000000..8057b6fa --- /dev/null +++ b/tests/waterdata_properties_test.py @@ -0,0 +1,89 @@ +"""Live monitor: the columns a getter documents are the ones its collection has. + +Some getters list their returned columns in the ``properties`` docstring, under +"Available options are:". The list is hand-written, so it goes stale when USGS +adds or removes a field. It is checked here rather than generated, because a +generated list would make the docs build depend on the live service and would +not reach ``help()``. When this fails, edit the docstring it names. +""" + +import inspect + +import httpx +import pytest + +from dataretrieval import waterdata +from dataretrieval.waterdata.endpoints import ogc_api_url + +#: Getters that list their returned columns, by collection. Written out rather +#: than discovered, so a getter that loses its list fails instead of dropping out. +_DOCUMENTED = { + "daily": waterdata.get_daily, + "continuous": waterdata.get_continuous, + "time-series-metadata": waterdata.get_time_series_metadata, +} + +_LABEL = "Available options are:" + +#: ``time-series-metadata`` lists ``id`` in its schema and the others do not, +#: but all three accept it, so it is excluded from both sides. +_ALWAYS_REQUESTABLE = {"id"} + +_TIMEOUT = 60 + + +def _documented_properties(getter) -> set[str]: + """The column names *getter* documents under :data:`_LABEL`. + + After ``inspect.getdoc`` dedents, the list runs from the label to the next + unindented line, which is the next numpydoc parameter. + """ + doc = inspect.getdoc(getter) or "" + start = doc.find(_LABEL) + assert start != -1, f"{getter.__name__} no longer documents its columns" + + lines = doc[start + len(_LABEL) :].splitlines() + collected = [lines[0]] + for line in lines[1:]: + if line.strip() and not line.startswith(" "): + break + collected.append(line) + return { + name.strip().rstrip(".") + for name in " ".join(collected).split(",") + if name.strip() + } + + +def _schema_properties(collection: str) -> set[str]: + """The columns *collection* publishes in its OGC schema document.""" + response = httpx.get( + f"{ogc_api_url()}/collections/{collection}/schema", + params={"f": "json"}, + timeout=_TIMEOUT, + ) + response.raise_for_status() + properties = response.json().get("properties") + assert properties, f"{collection} published no schema properties" + return set(properties) + + +@pytest.mark.live +@pytest.mark.parametrize("collection", sorted(_DOCUMENTED)) +def test_documented_columns_match_the_collection_schema(collection): + """A getter's documented column list matches what the collection publishes. + + On failure, edit the getter's ``properties`` docstring. For an added field, + consider a named parameter too; a removed field is a breaking change and + belongs in NEWS. + """ + getter = _DOCUMENTED[collection] + documented = _documented_properties(getter) - _ALWAYS_REQUESTABLE + published = _schema_properties(collection) - _ALWAYS_REQUESTABLE + + assert documented == published, ( + f"{getter.__name__} documents the wrong columns for {collection}: " + f"missing={sorted(published - documented)}, " + f"stale={sorted(documented - published)}. Edit the 'Available options " + "are:' list in its properties docstring." + ) diff --git a/tests/waterdata_queryables_test.py b/tests/waterdata_queryables_test.py index a0b2c262..1464cd0e 100644 --- a/tests/waterdata_queryables_test.py +++ b/tests/waterdata_queryables_test.py @@ -11,7 +11,7 @@ import httpx, json from typing import get_args from dataretrieval.waterdata.types import WATERDATA_SERVICES - base = "https://api.waterdata.usgs.gov/ogcapi/v0" + base = "https://api.waterdata.usgs.gov/ogcapi/v1" snap = {} for c in get_args(WATERDATA_SERVICES): r = httpx.get(f"{base}/collections/{c}/queryables", timeout=30) @@ -36,7 +36,7 @@ # The OGC queryables endpoint for any Water Data collection. QUERYABLES_RE = re.compile( - r"^https://api\.waterdata\.usgs\.gov/ogcapi/v0/collections/[^/]+/queryables$" + r"^https://api\.waterdata\.usgs\.gov/ogcapi/v1/collections/[^/]+/queryables$" ) # A minimal queryables document (the JSON Schema shape the real endpoint returns). @@ -99,10 +99,10 @@ def test_get_queryables_unknown_collection_raises(httpx_mock): # --- passthrough queryables (mocked) --------------------------------------- _DAILY_ITEMS_RE = re.compile( - r"^https://api\.waterdata\.usgs\.gov/ogcapi/v0/collections/daily/items" + r"^https://api\.waterdata\.usgs\.gov/ogcapi/v1/collections/daily/items" ) _DAILY_SCHEMA_RE = re.compile( - r"^https://api\.waterdata\.usgs\.gov/ogcapi/v0/collections/daily/schema$" + r"^https://api\.waterdata\.usgs\.gov/ogcapi/v1/collections/daily/schema$" ) _EMPTY_FEATURES = { "type": "FeatureCollection", diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 60c25e79..23d1b1fa 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -13,6 +13,8 @@ import pytest from pandas import DataFrame +import dataretrieval +from dataretrieval import configuration from dataretrieval.ogc.requests import ( _check_monitoring_location_id, _normalize_str_iterable, @@ -24,6 +26,7 @@ _construct_cql_request as _construct_cql_request_explicit, ) from dataretrieval.waterdata import ( + WaterdataConfiguration, get_channel, get_combined_metadata, get_continuous, @@ -49,7 +52,11 @@ _get_args, ) -_OGC_BASE = "https://api.waterdata.usgs.gov/ogcapi/v0" +_OGC_BASE = "https://api.waterdata.usgs.gov/ogcapi/v1" +#: The version the deprecated time-series-metadata filters are served from. +#: Spelled out rather than derived from ``_OGC_BASE``, so that it still names +#: v0 once the default moves on. +_V0_OGC_BASE = "https://api.waterdata.usgs.gov/ogcapi/v0" _STATS_BASE = "https://api.waterdata.usgs.gov/statistics/v0" #: Two real features per collection, captured from the live collection and trimmed. @@ -489,7 +496,7 @@ def test_construct_cql_request_post_verbatim_body(): assert req.method == "POST" assert req.headers["Content-Type"] == "application/query-cql-json" assert str(req.url).startswith( - "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" + "https://api.waterdata.usgs.gov/ogcapi/v1/collections/daily/items" ) # The body is sent through unchanged, not re-serialized. assert req.content.decode() == body @@ -646,11 +653,11 @@ def _schema_url(collection, *, base=_OGC_BASE): ) -def _mock_items(httpx_mock, collection, body=None, **kwargs): +def _mock_items(httpx_mock, collection, body=None, *, base=_OGC_BASE, **kwargs): """Serve ``collection``'s fixture for any ``/items`` request against it.""" httpx_mock.add_response( method=None, - url=_items_url(collection), + url=_items_url(collection, base=base), json=_fixture(collection) if body is None else body, **kwargs, ) @@ -898,6 +905,21 @@ def test_get_continuous(httpx_mock): assert "continuous_id" in df.columns assert df["time"].dtype.name.startswith("datetime64[") assert "UTC" in df["time"].dtype.name + # A code column stays the string the service sent. + assert df["method_category"].tolist() == ["UNKWN", "UNKWN"] + + +def test_get_continuous_sends_method_category(httpx_mock): + """The named ``method_category`` parameter reaches the request as a filter.""" + _mock_items(httpx_mock, "continuous") + + get_continuous( + monitoring_location_id="USGS-06904500", + method_category=["STNRD", "LMTUS"], + ) + + qs = _sent(httpx_mock, "continuous")[0] + assert qs["method_category"] == ["STNRD,LMTUS"] def test_get_latest_continuous(httpx_mock): @@ -1156,6 +1178,72 @@ def test_get_time_series_metadata(httpx_mock): assert qs["bbox"] == ["-89.840355,42.853411,-88.818626,43.422598"] +def test_time_series_metadata_goes_to_v1_without_an_advisory(httpx_mock): + """The default call goes to v1 and emits no DeprecationWarning.""" + _mock_items(httpx_mock, "time-series-metadata") + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + get_time_series_metadata( + monitoring_location_id="USGS-05427718", begin="1990-01-01/.." + ) + + (sent,) = httpx_mock.get_requests() + assert str(sent.url).startswith(f"{_OGC_BASE}/collections/time-series-metadata") + assert "begin=1990-01-01" in str(sent.url) + + +@pytest.mark.parametrize( + ("kwargs", "spelled"), + [ + ({"begin_utc": "1990-01-01/.."}, "begin_utc"), + ({"end_utc": "../2020-01-01"}, "end_utc"), + ({"state": "WI"}, "state"), + ({"state_name": "Wisconsin"}, "state_name"), + ({"hydrologic_unit_code": "07090002"}, "hydrologic_unit_code"), + ({"properties": ["begin_utc", "id"]}, "begin_utc"), + ], +) +def test_time_series_metadata_v0_only_filters_warn_and_go_to_v0( + httpx_mock, kwargs, spelled +): + """v1 dropped four filters and answers 400 or 500 to them. A call naming + one goes to v0, and the warning uses the caller's spelling.""" + _mock_items(httpx_mock, "time-series-metadata", base=_V0_OGC_BASE) + + with pytest.warns( + DeprecationWarning, + match=rf"'{spelled}' argument.*on or after 2027-06-01.*sent to v0", + ): + get_time_series_metadata(monitoring_location_id="USGS-05427718", **kwargs) + + (sent,) = httpx_mock.get_requests() + assert str(sent.url).startswith(f"{_V0_OGC_BASE}/collections/time-series-metadata") + + +def test_v0_routing_does_not_write_on_the_callers_configuration(httpx_mock): + """The getter sets the version on the request, not in the configuration. + + Entering a ``configure`` block would override a version the caller set, and + ``show_configuration()`` would report a version they never asked for. + """ + _mock_items(httpx_mock, "time-series-metadata", base=_V0_OGC_BASE) + _mock_items(httpx_mock, "daily") + + with dataretrieval.configure(WaterdataConfiguration(api_version="v1")): + with pytest.warns(DeprecationWarning): + get_time_series_metadata( + monitoring_location_id="USGS-05427718", begin_utc="1990-01-01/.." + ) + # The caller's setting is unchanged. + get_daily(monitoring_location_id="USGS-05427718") + assert configuration.api_version(adapter="waterdata") == "v1" + + legacy, other = (str(r.url) for r in httpx_mock.get_requests()) + assert legacy.startswith(f"{_V0_OGC_BASE}/collections/time-series-metadata") + assert other.startswith(f"{_OGC_BASE}/collections/daily") + + def test_get_combined_metadata(httpx_mock): _mock_items(httpx_mock, "combined-metadata") diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index f0d6eaf2..9eaf3176 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1153,11 +1153,11 @@ def test_403_without_an_envelope_names_the_credential_cause(): def test_error_messages_name_the_url(): """Without the URL a failed chunk in a fan-out cannot be traced back to the request that produced it -- the message is all the interruption holds.""" - request = httpx.Request("GET", "https://api.waterdata.usgs.gov/ogcapi/v0/x") + request = httpx.Request("GET", "https://api.waterdata.usgs.gov/ogcapi/v1/x") resp = httpx.Response(400, content=b"", request=request) with pytest.raises(HTTPError) as excinfo: _raise_for_non_200(resp) - assert "https://api.waterdata.usgs.gov/ogcapi/v0/x" in str(excinfo.value) + assert "https://api.waterdata.usgs.gov/ogcapi/v1/x" in str(excinfo.value) def test_error_message_survives_a_response_with_no_request():