diff --git a/NEWS.md b/NEWS.md index f4dbd033..002ad091 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +**09/22/2026:** Twenty columns that the Water Data OGC collections return are now named, documented parameters of the getter that returns them; before, they could be passed only through `**queryables`. `get_field_measurements()`: `control_condition`, `day`, `field_measurements_series_id`, `measurement_rated`, `month`, `reading_type`, `time_of_day`, `year`. `get_peaks()`: `qualifier`, `time_of_day`, `value`. `get_monitoring_locations()`: `revision_created`, `revision_modified`, `revision_note`. `get_combined_metadata()`: `data_gap_interval`, `reading_type`. `get_time_series_metadata()`: `data_gap_interval`, `parameter_description`. `get_field_measurements_metadata()`: `reading_type`. `get_channel()`: `channel_location_direction`. Existing calls send the same request as before. + +**09/22/2026:** `waterdata.get_continuous()` accepts `method_category`, a column USGS added to the `continuous` collection in September 2026: the RLMS method category code (`STNRD`, `LMTUS`, `EXPER` or `UNKWN`) for the method in effect over an observation's interval. It is returned on every record and is null for time series that have not been categorized. It could already be passed through `**queryables`; it is now a documented parameter. `get_latest_continuous()` is unchanged, because `latest-continuous` does not have the field. + +**09/22/2026:** The Water Data OGC getters now request **v1** of the Water Data APIs (`api.waterdata.usgs.gov/ogcapi/v1`), [released September 2026](https://waterdata.usgs.gov/blog/api-v1-release/); v0 stays online until June 2027. **New:** `WaterdataConfiguration(api_version="v0")`, or `api_version = "v0"` in the `[waterdata]` table of the configuration file, pins v0 during the transition. It changes only the version segment of the OGC path; Samples, Statistics and STAC are versioned separately and have no v1. **Behavior change:** `waterdata.get_time_series_metadata()` returns `begin` and `end` in UTC with a time zone, and no longer returns `begin_utc`, `end_utc`, `state_name` or `hydrologic_unit_code`. **Deprecation:** passing `begin_utc`, `end_utc`, `state`, `state_name` or `hydrologic_unit_code` to that getter, as a filter or in `properties`, emits a `DeprecationWarning` and sends the call to v0; this may be removed on or after 2027-06-01. Use `begin`, `end`, and `get_combined_metadata()` instead. **Behavior change:** `waterdata.get_field_measurements()` returns `time` as a date rather than a datetime, parsed to a tz-naive midnight timestamp as `get_daily()` already does; the time of day is in `time_of_day`. The `field-measurements-metadata` collection has no `time` field and is unaffected. + **09/09/2026:** **Bug fix:** code and identifier columns keep their leading zeros. A bare `pandas.read_csv` infers a zero-padded code as a number, so `waterdata.get_samples()` returned parameter code `00060` as `60` and HUC12 `070700050502` as `70700050502`, and `nwis.get_info()` returned `huc_cd` `02060005` as `2060005`. One rule now decides what a code column is — a name ending in `code`, the RDB abbreviation `_cd`, or a name containing `identifier`, `huc`, or `fips` — and every delimited response is parsed through it: the Samples and WQP CSV readers, `rdb.read_rdb` (which reads the names from the RDB header rather than the caller listing them), and the Water Use CSV pages. **Behavior change:** these columns now hold strings. `waterdata.get_samples()`: `USGSpcode`, `Location_HUCEightDigitCode`, `Location_HUCTwelveDigitCode`, `SampleCollectionMethod_Identifier` (`get_samples_summary()` shares the parse; no column in its current profile was affected). `nwis.get_info()`, `nwis.what_sites()`, and `nwis.get_record(service="site")`: `huc_cd`, `state_cd`, `county_cd`, `district_cd`. A comparison against a number — `df["USGSpcode"] == 60` — or a merge onto a numeric key now matches nothing instead of raising, so compare against the padded string (`== "00060"`) or call `.astype(int)` where the number is what you want. **Behavior change:** a count whose name reads as an identifier is numeric again. WQP's `AlternateLocation_IdentifierCount` has been read as text since 05/31/2026 because "Identifier" appears in its name; a name ending in `count` is now excluded from the rule, so the same column has one dtype in every service that reports it. Measurement columns are unchanged, and the `waterdata` OGC getters and `ngwmn` were never affected: their JSON responses deliver codes as strings and numeric coercion there is limited to a fixed list of measurement columns. **Correction to the 1.2.0 notes:** the same fix was applied to the nine `wqp` getters on 05/31/2026 and never recorded here — `wqp.get_results()` and the `what_*` getters have returned HUCs, parameter codes, and FIPS codes as strings since that release. **09/01/2026:** **Announcement:** We at USGS Water Data for the Nation want your feedback! Tell us how we're doing by taking our quick [survey](https://usgswaterresources.gov1.qualtrics.com/jfe/form/SV_07gX8G1DeOtVrH8), available through September 2026. diff --git a/dataretrieval/_configuration_core.py b/dataretrieval/_configuration_core.py index 62b3a6c4..f228e2d9 100644 --- a/dataretrieval/_configuration_core.py +++ b/dataretrieval/_configuration_core.py @@ -11,6 +11,7 @@ import math import os +import re import stat import sys import warnings @@ -26,11 +27,16 @@ from dataretrieval.exceptions import ConfigurationError #: Settings only an adapter can hold, because they name one service. No -#: package-wide value could mean anything for them: there is no one base URL. +#: package-wide value could mean anything for them: there is no one base URL, +#: and a version is a segment of one service's paths. #: #: The package-wide roster is :data:`SETTINGS`, declared below the class it is #: derived from. -ADAPTER_ONLY_SETTINGS: tuple[str, ...] = ("base_url",) +ADAPTER_ONLY_SETTINGS: tuple[str, ...] = ("base_url", "api_version") + +#: The adapter-only settings the file also refuses, so only a ``configure()`` +#: block can supply them (ADR 0011). +BLOCK_ONLY_SETTINGS: tuple[str, ...] = ("base_url",) #: Environment variable backing a setting (precedence step 2). #: @@ -52,12 +58,9 @@ #: Variables the environment is *refused* for, by setting. Named rather than left out of #: :data:`ENV_VARS`, so a caller who exports ``API_USGS_BASE_URL`` gets an error instead -#: of an ignored variable. The file refuses the same key in the same words -#: (:func:`_accepted_keys`): a base URL set outside the code could redirect the library -#: to another host without a reader of the script seeing it (ADR 0011). -#: -#: Derived from :data:`ADAPTER_ONLY_SETTINGS` so the file and the environment -#: cannot drift apart on which settings are code-only. +#: of an ignored variable. Derived from :data:`ADAPTER_ONLY_SETTINGS`, because a +#: variable applies to every adapter and those settings name one. The file +#: refuses only :data:`BLOCK_ONLY_SETTINGS` (ADR 0011). _REFUSED_ENV_VARS: dict[str, str] = { name: f"API_USGS_{name.upper()}" for name in ADAPTER_ONLY_SETTINGS } @@ -393,9 +396,7 @@ def _provenance(self) -> str: # Plain mixins rather than ``BaseConfiguration`` subclasses: a group has no # adapter and cannot be passed to :func:`configure`, so keeping it off that # branch leaves one linear base for the behavior. Frozen because a dataclass -# may not mix frozen and non-frozen bases; fields collect in reverse MRO order, -# so an adapter composing all four reads ``retries, stall_timeout, base_url, -# concurrency, parallel_chunks``. +# may not mix frozen and non-frozen bases. Fields collect in reverse MRO order. @dataclass(frozen=True) @@ -413,6 +414,13 @@ class _Redirectable: base_url: str | None = _UNSET +@dataclass(frozen=True) +class _Versioned: + """An adapter whose service publishes its API under a version path segment.""" + + api_version: str | None = _UNSET + + @dataclass(frozen=True) class _Concurrent: """An adapter that issues more than one request per call.""" @@ -771,6 +779,7 @@ def _coerce_count(value: object, label: str, optional: str) -> str: #: so the wider check cannot change a TOML outcome.) _TYPES: dict[str, Callable[[object, str, str], str]] = { "api_key": _coerce_string, + "api_version": _coerce_string, "base_url": _coerce_string, "progress": _coerce_progress, "concurrency": _coerce_concurrency, @@ -899,6 +908,25 @@ def _parse_base_url(raw: str, label: str) -> str: return value +#: A version path segment: ``v`` followed by digits, as in every Water Data path. +_API_VERSION_RE = re.compile(r"^v\d+$") + + +def _parse_api_version(raw: str, label: str) -> str: + """Parse an API version: the segment the service publishes it under, ``v1``. + + Checked as a shape, not against a list: a list here would refuse a version + the service already serves until a release of this package added it. + """ + value = raw.strip() + if not _API_VERSION_RE.match(value): + raise ConfigurationError( + f"{label} must be the version segment of the service's path, " + f"such as 'v1' (got {raw!r})." + ) + return value + + def _parse_progress(raw: str, label: str, *, strict: bool) -> bool: """Parse a progress toggle, optionally preserving legacy env truthiness.""" value = raw.strip().lower() @@ -931,6 +959,7 @@ def _parse_progress(raw: str, label: str, *, strict: bool) -> bool: "parallel_chunks": _parse_parallel_chunks, "stall_timeout": _parse_seconds, "base_url": _parse_base_url, + "api_version": _parse_api_version, } @@ -1048,11 +1077,13 @@ def _adapter_file_settings( where = f"[{adapter}]" # An adapter this process has not imported declares no vocabulary, so its - # table is checked against the package-wide settings alone: refusing a key - # for want of a schema would make the file's validity depend on which - # optional extras happened to be installed. + # table is checked against every setting this release has a grammar for: + # refusing a key for want of a schema would make the file's validity depend + # on which optional extras happened to be installed. accepted = settings_for(adapter) - validated = _scalars(table, path, where, SETTINGS if accepted is None else accepted) + validated = _scalars( + table, path, where, _ALL_SETTINGS if accepted is None else accepted + ) label = f"{path} {where}" result: Mapping[str, tuple[str, str]] = MappingProxyType( {name: (value, label) for name, value in validated.items()} @@ -1230,7 +1261,7 @@ def _accepted_keys( # and :func:`_named_profile` refuses a table inside a profile, so a # sub-table here is always a profile rather than deeper nesting. continue - if key in ADAPTER_ONLY_SETTINGS: + if key in BLOCK_ONLY_SETTINGS: # Rejected from the file wherever it appears. A file that # redirects a data-retrieval library to another host is a # supply-chain hazard; an in-code block keeps the redirect @@ -1240,16 +1271,7 @@ def _accepted_keys( "configure() block, never from a file." ) if key not in allowed: - if key in SETTINGS: - # A real setting, in a table that does not read it. Unlike an - # unrecognized name -- which may belong to a newer release -- - # this cannot become meaningful later, and ignoring it without an error - # would leave a caller believing they had tuned something. See - # ADR 0010. - raise ConfigurationError( - f"{path}: {key!r} at {where} is not a setting that table " - f"accepts. It accepts: {', '.join(sorted(allowed))}." - ) + _reject_known_setting(key, path, where, allowed) warnings.warn( f"{path}: unknown setting {key!r} at {where} (ignored). " f"Known settings: {', '.join(SETTINGS)}.", @@ -1261,6 +1283,31 @@ def _accepted_keys( return out +def _reject_known_setting( + key: str, path: Path, where: str, allowed: frozenset[str] | tuple[str, ...] +) -> None: + """Raise if *key* is a setting this release defines but that table cannot use. + + Returns for an unknown name, which the caller warns about instead: an + unknown name may belong to a newer release, but a known setting in the + wrong table will never take effect, and ignoring it would leave the caller + believing it had (ADR 0010). + """ + if where == _TOP_LEVEL and key in ADAPTER_ONLY_SETTINGS: + # The generic message below would list only top-level settings, none of + # which is the one to write. + raise ConfigurationError( + f"{path}: {key!r} at {where} names one service and has no " + "package-wide value; set it in the table of the adapter it belongs " + "to, such as [waterdata]." + ) + if key in _ALL_SETTINGS: + raise ConfigurationError( + f"{path}: {key!r} at {where} is not a setting that table " + f"accepts. It accepts: {', '.join(sorted(allowed))}." + ) + + def _checked_table( table: dict[str, Any], path: Path, diff --git a/dataretrieval/_deprecation.py b/dataretrieval/_deprecation.py index 4c8f1968..b4b12e94 100644 --- a/dataretrieval/_deprecation.py +++ b/dataretrieval/_deprecation.py @@ -19,6 +19,10 @@ "waterdata.get_cql(service=)": "2027-08-09", "wateruse": "2027-08-11", "ogc.interruptions": "2027-08-25", + # Set by the service, not by this package: v0 of the collection serves these + # filters until June 2027, and the shim stops working when that endpoint is + # retired (https://waterdata.usgs.gov/blog/api-v1-release/). + "waterdata.get_time_series_metadata(v0 filters)": "2027-06-01", } diff --git a/dataretrieval/configuration.py b/dataretrieval/configuration.py index c5b4be00..6ff8daf1 100644 --- a/dataretrieval/configuration.py +++ b/dataretrieval/configuration.py @@ -72,6 +72,7 @@ # isort: off from dataretrieval._configuration_core import ( ADAPTERS as ADAPTERS, + BLOCK_ONLY_SETTINGS as BLOCK_ONLY_SETTINGS, CONFIG_PATH_ENV as CONFIG_PATH_ENV, CONCURRENCY_UNBOUNDED as CONCURRENCY_UNBOUNDED, DEFAULT_CONCURRENCY as DEFAULT_CONCURRENCY, @@ -94,6 +95,7 @@ _Frame as _Frame, _named_profiles as _named_profiles, _NO_FILE as _NO_FILE, + _parse_api_version as _parse_api_version, _parse_base_url as _parse_base_url, _parse_concurrency as _parse_concurrency, _parse_parallel_chunks as _parse_parallel_chunks, @@ -112,6 +114,7 @@ _SettingValue as _SettingValue, _UNSET as _UNSET, _validated_raw as _validated_raw, + _Versioned as _Versioned, config_path as config_path, settings_for as settings_for, ) @@ -655,6 +658,41 @@ def base_url(*, adapter: str | None = None, default: str | None = None) -> str | return _parse_base_url(raw, label) +@overload +def api_version(*, adapter: str | None = ...) -> str | None: ... + + +@overload +def api_version(*, adapter: str | None = ..., default: str) -> str: ... + + +def api_version( + *, adapter: str | None = None, default: str | None = None +) -> str | None: + """An adapter's configured API version, falling back to *default*. + + Settable from code or from the adapter's table in the file. The + environment refuses it, as it refuses every adapter-only setting + (:data:`_REFUSED_ENV_VARS`). + + Like :func:`base_url`, it has no package-wide default. The adapter passes + its own, as in ``api_version(adapter="waterdata", default=OGC_API_VERSION)``, + so the version is declared in the module that builds the path. + + Parameters + ---------- + adapter : str, optional + Whose version to resolve. + default : str, optional + Returned when nothing configured a version. If omitted, ``None`` is + returned; :func:`show_configuration` relies on this. + """ + raw, label, _source = _resolve("api_version", adapter) + if raw is None: + return default + return _parse_api_version(raw, label) + + # --- resolution ---------------------------------------------------------- #: Which source of the chain supplied a resolution. Machine-readable so a @@ -724,7 +762,7 @@ def _check_adapter_known(adapter: str | None) -> None: def _check_env_not_refused(name: str) -> None: - """Raise if an environment variable is set for a code-only setting. + """Raise if an environment variable is set for an adapter-only setting. Refused before anything is consulted, not when the chain reaches the environment source. The file and the environment refuse ``base_url`` as one rule (ADR 0011), so @@ -732,13 +770,24 @@ def _check_env_not_refused(name: str) -> None: to work. """ refused = _REFUSED_ENV_VARS.get(name) - if refused is not None and refused in os.environ: - raise ConfigurationError( - f"{_env_label(refused)} is set, but {name!r} may only be set " - "in code, in a configure() block, never from the environment. Unset " - f"it and pass the value on the adapter's configuration, e.g. " - f"WaterdataConfiguration({name}=...)." - ) + if refused is None or refused not in os.environ: + return + block_only = name in BLOCK_ONLY_SETTINGS + fault = ( + "may only be set in code, in a configure() block, never from the environment" + if block_only + else "names one service and has no package-wide value, so the environment " + "cannot set it" + ) + # Suggest the file only for the settings it accepts. + or_the_file = ( + "" if block_only else ", or set the value in that adapter's table of the file" + ) + raise ConfigurationError( + f"{_env_label(refused)} is set, but {name!r} {fault}. Unset it and pass " + f"the value on the adapter's configuration, e.g. " + f"WaterdataConfiguration({name}=...){or_the_file}." + ) def _resolve_from_block( @@ -837,6 +886,7 @@ def _display_progress(_adapter: str | None = None) -> str: "parallel_chunks": lambda adapter: str(parallel_chunks(adapter=adapter)), "stall_timeout": lambda adapter: f"{stall_timeout(adapter=adapter):g}s", "base_url": lambda adapter: base_url(adapter=adapter) or "", + "api_version": lambda adapter: api_version(adapter=adapter) or "", } if set(_DISPLAYS) != set(_ALL_SETTINGS): # pragma: no cover - guards a coding error diff --git a/dataretrieval/waterdata/configuration.py b/dataretrieval/waterdata/configuration.py index eb353570..26f5915a 100644 --- a/dataretrieval/waterdata/configuration.py +++ b/dataretrieval/waterdata/configuration.py @@ -18,6 +18,7 @@ _Redirectable, _register, _Retrying, + _Versioned, ) __all__ = ["WaterdataConfiguration"] @@ -25,7 +26,7 @@ @dataclass(frozen=True) class WaterdataConfiguration( - _Chunked, _Concurrent, _Redirectable, _Retrying, BaseConfiguration + _Chunked, _Concurrent, _Redirectable, _Retrying, _Versioned, BaseConfiguration ): """Settings for Water Data calls alone. @@ -45,9 +46,17 @@ class WaterdataConfiguration( base_url : str, optional Root to send Water Data requests to, instead of the service's own. The package appends its own paths, so one value redirects all four families together -- - ``/ogcapi/v0``, ``/samples-data``, ``/statistics/v0`` and ``/stac/v0``. Code - only: the file and the environment refuse it. The API key is scoped to the host - that accepts it, so a redirected call sends no key. + ``/ogcapi/``, ``/samples-data``, ``/statistics/v0`` and + ``/stac/v0``. Code only: the file and the environment refuse it. The API key is + scoped to the host that accepts it, so a redirected call sends no key. + api_version : str, optional + Version of the Water Data API to request, as the segment of its path: + ``"v1"``, which this release is written against, or ``"v0"`` while the + service keeps it online (until June 2027). It replaces that one segment, + so the Samples, Statistics and STAC families -- versioned separately, with + no v1 -- are unaffected. Settable in code or in the ``[waterdata]`` table + of the file, never from the environment. Under another version, getters + return that version's columns as the service sends them. concurrency : int or str, optional Cap on simultaneous sub-requests, or ``"unbounded"``. parallel_chunks : int, optional @@ -56,11 +65,11 @@ class WaterdataConfiguration( """ # The settings this service reads, named by the groups they come from: - # every adapter's retry settings, a redirectable base, and -- because Water - # Data queries divide along a URL byte budget and are executed concurrently - # -- both fan-out settings. Each group declares the setting itself once, in - # :mod:`dataretrieval.configuration`, which also defines its grammar and - # its coercion. + # every adapter's retry settings, a redirectable base, a versioned API, and + # -- because Water Data queries divide along a URL byte budget and are + # executed concurrently -- both fan-out settings. Each group declares the + # setting itself once, in :mod:`dataretrieval.configuration`, which also + # defines its grammar and its coercion. adapter: ClassVar[str] = "waterdata" diff --git a/dataretrieval/waterdata/endpoints.py b/dataretrieval/waterdata/endpoints.py index 8c153b87..81da1300 100644 --- a/dataretrieval/waterdata/endpoints.py +++ b/dataretrieval/waterdata/endpoints.py @@ -11,10 +11,16 @@ from dataretrieval import configuration as _configuration from dataretrieval.credentials import WATERDATA_BASE_URL +#: The version of the Water Data OGC API this release requests by default and +#: shapes responses for. +OGC_API_VERSION = "v1" + #: Canonical paths below the Water Data root. They are not endpoints on their #: own: callers obtain complete destinations through the request-time functions -#: below (ADR 0011). -_OGC_API_PATH = "/ogcapi/v0" +#: below (ADR 0011). Only the OGC family takes its version at request time; the +#: Statistics service and the STAC catalog publish their own versions and have +#: no v1 (checked 2026-09-22). +_OGC_API_PATH = "/ogcapi" _SAMPLES_PATH = "/samples-data" _STATISTICS_API_PATH = "/statistics/v0" _RATINGS_CATALOG_PATH = "/stac/v0" @@ -22,7 +28,7 @@ # Default-value compatibility for the documented ``waterdata.utils`` constants. # Production collection-family modules do not import these raw values. _DEFAULT_BASE_URL = WATERDATA_BASE_URL -_DEFAULT_OGC_API_URL = f"{_DEFAULT_BASE_URL}{_OGC_API_PATH}" +_DEFAULT_OGC_API_URL = f"{_DEFAULT_BASE_URL}{_OGC_API_PATH}/{OGC_API_VERSION}" _DEFAULT_SAMPLES_URL = f"{_DEFAULT_BASE_URL}{_SAMPLES_PATH}" @@ -32,9 +38,23 @@ def _endpoint(path: str) -> str: return f"{root}{path}" -def ogc_api_url() -> str: - """Return the OGC collections endpoint for the effective configuration.""" - return _endpoint(_OGC_API_PATH) +def ogc_api_url(api_version: str | None = None) -> str: + """Return the OGC collections endpoint for the effective configuration. + + Parameters + ---------- + api_version : str, optional + Version for this one request, used by a getter that must reach a + version other than the configured one. ``None`` (the default) resolves + the version through the configuration chain. A getter passes this + instead of entering a ``configure`` block, because a block set by the + library would override the caller's own setting (ADR 0011). + """ + if api_version is None: + api_version = _configuration.api_version( + adapter="waterdata", default=OGC_API_VERSION + ) + return _endpoint(f"{_OGC_API_PATH}/{api_version}") def samples_url() -> str: @@ -53,6 +73,7 @@ def ratings_catalog_url() -> str: __all__ = [ + "OGC_API_VERSION", "ogc_api_url", "ratings_catalog_url", "samples_url", diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py index 5060a267..409bd3eb 100644 --- a/dataretrieval/waterdata/measurements.py +++ b/dataretrieval/waterdata/measurements.py @@ -26,6 +26,14 @@ def get_field_measurements( monitoring_location_id: str | Iterable[str] | None = None, parameter_code: str | Iterable[str] | None = None, observing_procedure_code: str | Iterable[str] | None = None, + control_condition: str | Iterable[str] | None = None, + day: int | list[int] | None = None, + field_measurements_series_id: str | Iterable[str] | None = None, + measurement_rated: str | Iterable[str] | None = None, + month: int | list[int] | None = None, + reading_type: str | Iterable[str] | None = None, + time_of_day: str | Iterable[str] | None = None, + year: int | list[int] | None = None, properties: str | Iterable[str] | None = None, field_visit_id: str | Iterable[str] | None = None, approval_status: str | Iterable[str] | None = None, @@ -69,12 +77,35 @@ def get_field_measurements( observing_procedure_code : string or iterable of strings, optional A short code corresponding to the observing procedure for the field measurement. + control_condition : string or iterable of strings, optional + The state of the control feature at the time of observation. + day : integer or list of integers, optional + The day of the month the field measurement was taken. If null, the day is + unknown. + field_measurements_series_id : string or iterable of strings, optional + A unique identifier representing a single collection series, corresponding to + the id field in the field-measurements-metadata endpoint. A collection series + is the set of field measurements at one monitoring location for a single + parameter code using a single reading type. + measurement_rated : string or iterable of strings, optional + A qualitative estimate of the quality of a measurement. + month : integer or list of integers, optional + The calendar month the field measurement was taken. If null, the month is + unknown. + reading_type : string or iterable of strings, optional + Distinguishes field-measurement readings from measurements. Readings have a + value of ReferencePrimary; measurements are Discharge or MeanGageHeight. + time_of_day : string or iterable of strings, optional + The time of day the field measurement was taken. If null, the time of day is + unknown. The time column holds only the date. + year : integer or list of integers, optional + The calendar year the field measurement was taken. properties : string or iterable of strings, optional The columns to return from the query. See the field-measurements schema in the OpenAPI reference for the available columns (e.g. geometry, id, monitoring_location_id, parameter_code, value, unit_of_measure, approval_status, qualifier, last_modified): - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/field-measurements field_visit_id : string or iterable of strings, optional A universally unique identifier (UUID) for the field visit. Multiple measurements may be made during a single field visit. @@ -240,6 +271,9 @@ def get_peaks( month: int | list[int] | None = None, day: int | list[int] | None = None, peak_since: int | list[int] | None = None, + qualifier: str | Iterable[str] | None = None, + time_of_day: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, properties: str | Iterable[str] | None = None, skip_geometry: bool | None = None, bbox: list[float] | None = None, @@ -261,7 +295,7 @@ def get_peaks( The collection covers both stage (parameter ``"00065"``, ``ft``) and discharge (parameter ``"00060"``, ``ft^3/s``); a typical streamgage has a series for each. Reference docs: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/peaks + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/peaks Parameters ---------- @@ -288,6 +322,15 @@ def get_peaks( peak_since : int or list of ints, optional Filter on the year since which the peak value has been the record (the API serves this field as an integer; many rows are ``null``). + qualifier : string or iterable of strings, optional + Any qualifiers associated with a peak, for instance whether a sensor may have + been impacted by ice or whether the value was estimated. + time_of_day : string or iterable of strings, optional + The time of day a peak occurred. If null, the time of day is unknown, as is + common for historical peaks recorded only to the day. + value : string or iterable of strings, optional + The value of the peak. Values are transmitted as strings in the JSON response + to preserve precision. properties : string or iterable of strings, optional Subset of columns to return. Defaults to every available property. skip_geometry : boolean, optional @@ -393,6 +436,7 @@ def get_channel( measurement_type: str | Iterable[str] | None = None, last_modified: str | Iterable[str] | None = None, channel_measurement_type: str | Iterable[str] | None = None, + channel_location_direction: str | Iterable[str] | None = None, properties: str | Iterable[str] | None = None, skip_geometry: bool | None = None, bbox: list[float] | None = None, @@ -494,6 +538,8 @@ def get_channel( Water Data APIs use camelCase "skipGeometry" in CQL2 queries. channel_measurement_type : string or iterable of strings, optional The channel measurement type. + channel_location_direction : string or iterable of strings, optional + Location of the measurement from the gage. properties : string or iterable of strings, optional The columns to return from the query. Available options are: geometry, channel_measurements_id, monitoring_location_id, @@ -503,8 +549,8 @@ def get_channel( channel_location_distance, channel_location_distance_unit, channel_stability, channel_material, channel_evenness, horizontal_velocity_description, vertical_velocity_description, longitudinal_velocity_description, - measurement_type, last_modified, channel_measurement_type. The default - (None) returns all columns. + measurement_type, last_modified, channel_measurement_type, + channel_location_direction. The default (None) returns all columns. bbox : list of numbers, optional Only features whose geometry intersects the bounding box are selected. The bounding box is provided as four or six numbers, depending on diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py index eb26d98e..ece6473b 100644 --- a/dataretrieval/waterdata/metadata.py +++ b/dataretrieval/waterdata/metadata.py @@ -13,6 +13,7 @@ import pandas as pd +from dataretrieval._deprecation import REMOVALS, warn_deprecated from dataretrieval.waterdata.utils import ( _get_args, _with_state, @@ -66,6 +67,9 @@ def get_monitoring_locations( well_constructed_depth: str | Iterable[str] | None = None, hole_constructed_depth: str | Iterable[str] | None = None, depth_source_code: str | Iterable[str] | None = None, + revision_created: str | Iterable[str] | None = None, + revision_modified: str | Iterable[str] | None = None, + revision_note: str | Iterable[str] | None = None, properties: str | Iterable[str] | None = None, skip_geometry: bool | None = None, bbox: list[float] | None = None, @@ -249,6 +253,14 @@ def get_monitoring_locations( A code indicating the source of water-level data. A `list of codes `_ is available. + revision_created : string or iterable of strings, optional + The date a revision statement was created. + revision_modified : string or iterable of strings, optional + The most recent date a revision statement was modified. + revision_note : string or iterable of strings, optional + Text explaining a revision to this location's approved data. Revisions are + also flagged by revision qualifier codes in the data. Explanations from + before 2017 may not be online but can be requested. properties : string or iterable of strings, optional The columns to return from the query. Available options are: geometry, id, agency_code, agency_name, @@ -264,7 +276,8 @@ def get_monitoring_locations( contributing_drainage_area, time_zone_abbreviation, uses_daylight_savings, construction_date, aquifer_code, national_aquifer_code, aquifer_type_code, well_constructed_depth, - hole_constructed_depth, depth_source_code. + hole_constructed_depth, depth_source_code, revision_created, + revision_modified, revision_note. bbox : list of numbers, optional Only features whose geometry intersects the bounding box are selected. The bounding box is provided as four or six numbers, depending on @@ -343,10 +356,55 @@ def get_monitoring_locations( return get_ogc_data(args, collection, max_rows=max_rows) +#: Filters that v1 of ``time-series-metadata`` dropped (it responds with 400 or +#: 500 to them, as a filter or in ``properties``), mapped to what to use +#: instead. ``state`` resolves to ``state_name`` before the check, so that entry +#: covers both. +#: +#: The renames do not use ``_accept_legacy_kwargs`` (ADR 0012) because the old +#: name may also appear in ``properties``, and translating it would rename a +#: column the caller asked for by name. +_V0_ONLY_FILTERS: dict[str, str] = { + "begin_utc": "'begin'", + "end_utc": "'end'", + "state_name": "get_combined_metadata(state=...)", + "hydrologic_unit_code": "get_combined_metadata(hydrologic_unit_code=...)", +} + + +def _time_series_metadata( + args: dict[str, Any], *, max_rows: int | None, state_given: bool +) -> tuple[pd.DataFrame, BaseMetadata]: + """Send the query, to v0 if it names a filter or ``properties`` column that + v1 dropped.""" + requested = set(args.get("properties", ())) + legacy = [n for n in _V0_ONLY_FILTERS if n in args or n in requested] + for name in legacy: + spelled = "state" if name == "state_name" and state_given else name + warn_deprecated( + f"The {spelled!r} argument of get_time_series_metadata", + replacement=_V0_ONLY_FILTERS[name], + removal=REMOVALS["waterdata.get_time_series_metadata(v0 filters)"], + detail=( + "v1 of the Water Data API has no such field, so this call is " + "sent to v0." + ), + stacklevel=3, + ) + return get_ogc_data( + args, + "time-series-metadata", + max_rows=max_rows, + api_version="v0" if legacy else None, + ) + + def get_time_series_metadata( monitoring_location_id: str | Iterable[str] | None = None, parameter_code: str | Iterable[str] | None = None, parameter_name: str | Iterable[str] | None = None, + data_gap_interval: str | Iterable[str] | None = None, + parameter_description: str | Iterable[str] | None = None, properties: str | Iterable[str] | None = None, statistic_id: str | Iterable[str] | None = None, hydrologic_unit_code: str | Iterable[str] | None = None, @@ -360,6 +418,7 @@ def get_time_series_metadata( unit_of_measure: str | Iterable[str] | None = None, computation_period_identifier: str | Iterable[str] | None = None, computation_identifier: str | Iterable[str] | None = None, + statistics_begin: str | Iterable[str] | None = None, thresholds: float | list[float] | None = None, sublocation_identifier: str | Iterable[str] | None = None, primary: str | Iterable[str] | None = None, @@ -399,20 +458,30 @@ def get_time_series_metadata( available at https://help.waterdata.usgs.gov/codes-and-parameters/parameters. parameter_name : string or iterable of strings, optional A human-understandable name corresponding to parameter_code. + data_gap_interval : string or iterable of strings, optional + The time interval threshold used for gap detection in the time series, as an + ISO 8601 duration. + parameter_description : string or iterable of strings, optional + A description of what the parameter code represents, as used by WDFN and other + USGS data dissemination products. properties : string or iterable of strings, optional The columns to return from the query. - Available options are: begin, begin_utc, computation_identifier, - computation_period_identifier, end, end_utc, geometry, - hydrologic_unit_code, id, last_modified, monitoring_location_id, - parameter_code, parameter_description, parameter_name, - parent_time_series_id, primary, state_name, statistic_id, - sublocation_identifier, thresholds, unit_of_measure, web_description + Available options are: begin, computation_identifier, + computation_period_identifier, data_gap_interval, end, geometry, id, + last_modified, monitoring_location_id, parameter_code, + parameter_description, parameter_name, parent_time_series_id, primary, + statistic_id, statistics_begin, sublocation_identifier, thresholds, + unit_of_measure, web_description statistic_id : string or iterable of strings, optional A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. hydrologic_unit_code : string or iterable of strings, optional + Deprecated: v1 of the Water Data API does not filter this collection by + hydrologic unit. A call that passes it is sent to v0, with a + ``DeprecationWarning``, until v0 is retired in June 2027. Use + :func:`get_combined_metadata` with ``hydrologic_unit_code`` instead. A unique hydrologic unit code (HUC) of two to eight digits, based on the four levels of classification in the hydrologic unit system. The United States is divided and sub-divided into successively smaller hydrologic @@ -421,12 +490,13 @@ def get_time_series_metadata( each other, from the smallest (cataloging units) to the largest (regions). state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"``). + Deprecated, as ``hydrologic_unit_code`` is: v1 does not filter this + collection by state. Use :func:`get_combined_metadata` with ``state`` + instead. Accepts a full name (``"Wisconsin"``), a two-letter postal + code (``"WI"``), or a two-digit ANSI/FIPS code (``"55"``). state_name : string or iterable of strings, optional - The name of the state or state equivalent in which the monitoring location - is located. + Deprecated; see ``state``. The name of the state or state equivalent in + which the monitoring location is located. last_modified : string, optional The last time a record was refreshed in our database. A refresh may happen due to regular operational processes and does not necessarily @@ -445,23 +515,14 @@ def get_time_series_metadata( for the last 36 hours begin : string or iterable of strings, optional - This field contains the same information as "begin_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - end : string or iterable of strings, optional - This field contains the same information as "end_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - begin_utc : string or iterable of strings, optional - The datetime of the earliest observation in the time series. Together - with end, this field represents the period of record of a time series. - Note that some time series may have large gaps in their collection - record. This field is currently in the local time of the monitoring - location. We intend to update this in version v0 to use UTC with a time - zone. You can query this field using date-times or intervals, adhering - to RFC 3339, or using ISO 8601 duration objects. Intervals may be - bounded or half-bounded (double-dots at start or end). Only features - that have a begin that intersects the value of datetime are selected. + The datetime of the earliest observation in the time series, in UTC + with a time zone. Together with end, this field represents the period + of record of a time series. Note that some time series may have large + gaps in their collection record. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a begin that intersects the + value of datetime are selected. Examples: * A date-time: "2018-02-12T23:20:50Z" @@ -471,19 +532,18 @@ def get_time_series_metadata( * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours - end_utc : string or iterable of strings, optional - The datetime of the most recent observation in the time series. Data returned by - this endpoint updates at most once per day, and potentially less frequently than - that, and as such there may be more recent observations within a time series - than the time series end value reflects. Together with begin, this field - represents the period of record of a time series. It is additionally used to - determine whether a time series is "active". We intend to update this in - version v0 to use UTC with a time zone. - You can query this field using date-times or intervals, - adhering to RFC 3339, or using ISO 8601 duration objects. Intervals - may be bounded or half-bounded (double-dots at start or end). Only - features that have an end that intersects the value of datetime are - selected. + end : string or iterable of strings, optional + The datetime of the most recent observation in the time series, in UTC + with a time zone. Data returned by this endpoint updates at most once + per day, and potentially less frequently than that, and as such there + may be more recent observations within a time series than the time + series end value reflects. Together with begin, this field represents + the period of record of a time series. It is additionally used to + determine whether a time series is "active". You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have an end that intersects the + value of datetime are selected. Examples: * A date-time: "2018-02-12T23:20:50Z" @@ -493,6 +553,12 @@ def get_time_series_metadata( * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours + begin_utc : string or iterable of strings, optional + Deprecated, as ``hydrologic_unit_code`` is: in v1, ``begin`` holds this + value. + end_utc : string or iterable of strings, optional + Deprecated, as ``hydrologic_unit_code`` is: in v1, ``end`` holds this + value. unit_of_measure : string or iterable of strings, optional A human-readable description of the units of measurement associated with an observation. @@ -501,6 +567,12 @@ def get_time_series_metadata( computation_identifier : string or iterable of strings, optional Indicates whether the data from this time series represent a specific statistical computation. + statistics_begin : string or iterable of strings, optional + The year from which statistics are computed for this time series. When + it is populated, WDFN ignores data before that year when computing + statistics; when it is empty, the full period of record is used. + Published by v1 only, so a call routed to v0 by one of the deprecated + filters above cannot use it. thresholds : number or list of numbers, optional Thresholds represent known numeric limits for a time series, for example the historic maximum value for a parameter or a level below which a @@ -583,15 +655,13 @@ def get_time_series_metadata( ... begin="1990-01-01/..", ... ) """ - collection = "time-series-metadata" - # Build argument dictionary, omitting None values (resolving the unified # `state` argument into the OGC `state_name` queryable). args = _get_args( _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} ) - return get_ogc_data(args, collection, max_rows=max_rows) + return _time_series_metadata(args, max_rows=max_rows, state_given=state is not None) def get_combined_metadata( @@ -651,6 +721,8 @@ def get_combined_metadata( well_constructed_depth: str | Iterable[str] | None = None, hole_constructed_depth: str | Iterable[str] | None = None, depth_source_code: str | Iterable[str] | None = None, + data_gap_interval: str | Iterable[str] | None = None, + reading_type: str | Iterable[str] | None = None, properties: str | Iterable[str] | None = None, skip_geometry: bool | None = None, bbox: list[float] | None = None, @@ -674,7 +746,7 @@ def get_combined_metadata( of record, …) in a single query. See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/combined-metadata + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/combined-metadata All ~35 location-catalog kwargs are accepted (``agency_code``, ``state_name``, ``drainage_area``, ``aquifer_code``, …) but only @@ -741,6 +813,12 @@ def get_combined_metadata( altitude, vertical/horizontal datum, drainage area, aquifer, well construction, …); see :func:`get_monitoring_locations` for descriptions of each. + data_gap_interval : string or iterable of strings, optional + The time interval threshold used for gap detection in the time series, as an + ISO 8601 duration. + reading_type : string or iterable of strings, optional + Distinguishes field-measurement readings from measurements. Readings have a + value of ReferencePrimary; measurements are Discharge or MeanGageHeight. properties : string or iterable of strings, optional Subset of columns to return. Defaults to every available property. @@ -860,6 +938,7 @@ def get_field_measurements_metadata( begin: str | Iterable[str] | None = None, end: str | Iterable[str] | None = None, last_modified: str | Iterable[str] | None = None, + reading_type: str | Iterable[str] | None = None, properties: str | Iterable[str] | None = None, skip_geometry: bool | None = None, bbox: list[float] | None = None, @@ -883,7 +962,7 @@ def get_field_measurements_metadata( field-measurement parameters a site has, and over what date range. See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements-metadata + https://api.waterdata.usgs.gov/ogcapi/v1/openapi?f=html#/field-measurements-metadata Parameters ---------- @@ -902,6 +981,9 @@ def get_field_measurements_metadata( interval (``"start/end"``, optionally half-bounded with ``..``), or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See :func:`get_time_series_metadata` for the full grammar. + reading_type : string or iterable of strings, optional + Distinguishes field-measurement readings from measurements. Readings have a + value of ReferencePrimary; measurements are Discharge or MeanGageHeight. properties : string or iterable of strings, optional Subset of columns to return. Defaults to every available property. skip_geometry : boolean, optional diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index c83232f0..e623c634 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -267,6 +267,7 @@ def get_continuous( approval_status: str | Iterable[str] | None = None, unit_of_measure: str | Iterable[str] | None = None, qualifier: str | Iterable[str] | None = None, + method_category: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, last_modified: str | Iterable[str] | None = None, time: str | Iterable[str] | None = None, @@ -316,7 +317,8 @@ def get_continuous( The columns to return from the query. Available options are: geometry, id, time_series_id, monitoring_location_id, parameter_code, statistic_id, time, value, - unit_of_measure, approval_status, qualifier, last_modified + unit_of_measure, approval_status, qualifier, method_category, + last_modified time_series_id : string or iterable of strings, optional A unique identifier representing a single time series, corresponding to the id field in the time-series-metadata endpoint. @@ -345,6 +347,12 @@ def get_continuous( qualifier : string or iterable of strings, optional Any qualifiers associated with an observation, for instance whether a sensor may have been impacted by ice or whether values were estimated. + method_category : string or iterable of strings, optional + The RLMS method category code for the method in effect over the + observation's interval: "STNRD" (standardized, with known uncertainty + and full QA/QC), "LMTUS" (limited use: a modified or externally + sourced method), "EXPER" (experimental), or "UNKWN" (unknown). + Null for time series that have not been categorized. value : string or iterable of strings, optional The value of the observation. Values are transmitted as strings in the JSON response format to preserve precision. @@ -947,7 +955,7 @@ def get_stats_por( site_type_code: string, optional Site type code query parameter. A list of valid site type codes is available at - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + https://api.waterdata.usgs.gov/ogcapi/v1/collections/site-types/items. Example: "GW" (Groundwater site) site_type_name: string, optional Site type name query parameter. @@ -1089,12 +1097,12 @@ def get_stats_date_range( site_type_code: string, optional Site type code query parameter. A list of valid site type codes is available at - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + https://api.waterdata.usgs.gov/ogcapi/v1/collections/site-types/items. Example: "GW" (Groundwater site) site_type_name: string, optional Site type name query parameter. A list of valid site type names is available at - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + https://api.waterdata.usgs.gov/ogcapi/v1/collections/site-types/items. Example: "Well" parameter_code : string or iterable of strings, optional A 5-digit code identifying the constituent measured and the units of diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index ede2e117..0e9270ac 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -197,6 +197,7 @@ def get_ogc_data( cql_body: str | None = None, *, spatial: bool = True, + api_version: str | None = None, ) -> tuple[pd.DataFrame, BaseMetadata]: """Water-Data wrapper over :func:`~dataretrieval.ogc.get_ogc_data`. @@ -226,6 +227,10 @@ def get_ogc_data( spatial : bool, optional Whether the collection includes feature geometry. Water Data's typed feature collections do; reference tables pass ``False``. + api_version : str, optional + API version for this one request. ``None`` (the default) resolves it + through the configuration chain. See + :func:`~dataretrieval.waterdata.endpoints.ogc_api_url`. Returns ------- @@ -245,7 +250,7 @@ def get_ogc_data( # The endpoint is resolved from the active ContextVar at request time; # the documented ``OGC_API_URL`` constant remains the default-value # compatibility path rather than a production request destination. - base_url=ogc_api_url(), + base_url=ogc_api_url(api_version), spatial=spatial, extra_id_cols=_EXTRA_ID_COLS, dialect=WATERDATA_DIALECT, diff --git a/demos/USGS_WaterData_DailyStatistics_Examples.ipynb b/demos/USGS_WaterData_DailyStatistics_Examples.ipynb index c4680eb2..c1aacc9d 100644 --- a/demos/USGS_WaterData_DailyStatistics_Examples.ipynb +++ b/demos/USGS_WaterData_DailyStatistics_Examples.ipynb @@ -389,7 +389,7 @@ "## Statistics API tips\n", "\n", "The statistics API does **not** follow the OGC standards used by the\n", - "`api.waterdata.usgs.gov/ogcapi/v0/` endpoints. A few things to keep in mind:\n", + "`api.waterdata.usgs.gov/ogcapi/v1/` endpoints. A few things to keep in mind:\n", "\n", "- **Higher rate limits.** At the time of writing the statistics API allows ~4000\n", " requests/hour per IP (per token if a token is supplied).\n", diff --git a/demos/USGS_WaterData_Introduction_Examples.ipynb b/demos/USGS_WaterData_Introduction_Examples.ipynb index 0e1254a0..51cc60bf 100644 --- a/demos/USGS_WaterData_Introduction_Examples.ipynb +++ b/demos/USGS_WaterData_Introduction_Examples.ipynb @@ -7,7 +7,7 @@ "source": [ "# Introduction to the USGS Water Data APIs\n", "\n", - "The [USGS Water Data APIs](https://api.waterdata.usgs.gov/ogcapi/v0/) are the\n", + "The [USGS Water Data APIs](https://api.waterdata.usgs.gov/ogcapi/v1/) are the\n", "modern, OGC-based replacement for the legacy NWIS web services. In Python they are\n", "exposed through the `dataretrieval.waterdata` module, which will gradually replace\n", "the older `dataretrieval.nwis` functions.\n", @@ -86,7 +86,7 @@ "\n", "Use the `properties` argument to choose which columns come back. The full set of\n", "available properties for a collection is published in that collection's schema,\n", - "e.g. ." + "e.g. ." ] }, { @@ -164,7 +164,7 @@ "\n", "`get_monitoring_locations` returns site metadata. To browse the service in a\n", "web browser, visit\n", - ".\n", + ".\n", "\n", "A simple request for one known USGS site:" ] @@ -291,7 +291,7 @@ "### Daily values\n", "\n", "`get_daily` returns daily values. Browse it at\n", - "." + "." ] }, { @@ -346,7 +346,7 @@ "### Continuous\n", "\n", "`get_continuous` returns instantaneous (sensor) values. Browse it at\n", - ".\n", + ".\n", "\n", "This service currently allows at most **3 years** of data per request; with no\n", "`time` argument it returns the latest year. Continuous data have no geometry\n", diff --git a/demos/WaterData_demo.ipynb b/demos/WaterData_demo.ipynb index ec6518fb..35b0cc20 100644 --- a/demos/WaterData_demo.ipynb +++ b/demos/WaterData_demo.ipynb @@ -616,7 +616,7 @@ "The USGS Water Data APIs belong to the Water Data for the Nation (WDFN) group of applications and tools. These products exist under the broader National Water Information System (NWIS) program. Check out the links below for more information on the USGS Water Data APIs and other ways to download or view USGS water data:\n", "* [Water Data APIs Home](https://api.waterdata.usgs.gov/)\n", "* [Get an API Key](https://api.waterdata.usgs.gov/signup/)\n", - "* [Water Data API OGC Endpoint Catalog](https://api.waterdata.usgs.gov/ogcapi/v0/collections?f=html)\n", + "* [Water Data API OGC Endpoint Catalog](https://api.waterdata.usgs.gov/ogcapi/v1/collections?f=html)\n", "* [Water Data Download Form](https://api.waterdata.usgs.gov/download)\n", "* [Water Data for the Nation Home](https://waterdata.usgs.gov/)\n", "* [Water Data for the Nation Feedback Form](https://waterdata.usgs.gov/questions-comments/?referrerUrl=https://api.waterdata.usgs.gov)\n", diff --git a/docs/source/architecture/decisions/0011-configuration-profiles.rst b/docs/source/architecture/decisions/0011-configuration-profiles.rst index 05ec2473..218378ea 100644 --- a/docs/source/architecture/decisions/0011-configuration-profiles.rst +++ b/docs/source/architecture/decisions/0011-configuration-profiles.rst @@ -226,7 +226,7 @@ Satisfied. In ``tests/configuration_test.py``: isolation rule under lazy validation. - ``test_base_url_applies_from_code_and_is_refused_from_the_file``, with ``test_base_url_is_refused_from_the_environment`` for the other source, and - ``test_a_code_base_url_redirects_every_water_data_endpoint_family`` -- one + ``test_base_url_changes_every_endpoint_family_and_api_version_only_ogc`` -- one public-getter contract covering OGC, Samples, Statistics, and Ratings. The endpoint module exposes functions that build each URL at request time rather than ready-made endpoint constants, so a family module gets the root in effect @@ -257,3 +257,10 @@ Notes - The setting-group clause was added after the original decision, consolidating under ADR 0000 a rule the configuration core was stating in prose. It does not change behavior. +- ``api_version`` was added on 2026-09-22 as a second adapter-only setting, + when the Water Data OGC collections moved to v1. Unlike ``base_url``, the + file accepts it: the code-only rule above exists because a base URL can + redirect requests to another host, and a version cannot. The file's refusal + is therefore keyed on a separate ``BLOCK_ONLY_SETTINGS`` list, while the + environment still refuses every adapter-only setting, because a variable is + package-wide. diff --git a/docs/source/userguide/configuration.rst b/docs/source/userguide/configuration.rst index d6a5d102..647962ff 100644 --- a/docs/source/userguide/configuration.rst +++ b/docs/source/userguide/configuration.rst @@ -134,6 +134,12 @@ Settings a ``configure`` block: a file that redirected the library to another host would be a supply-chain hazard. See :ref:`configuration-redirect`. + * - ``api_version`` + - the version the release targets + - *(none — code or file)* + - Which version of one service's API to request, as the segment of its + path (``"v1"``). Per adapter; Water Data reads it for its OGC + collections. See :ref:`configuration-api-version`. Where settings come from @@ -279,8 +285,8 @@ Adapter Configuration Ac ==================================== ====================================== ======================================== ``waterdata`` ``waterdata.WaterdataConfiguration`` ``concurrency``, ``parallel_chunks``, ``retries``, ``stall_timeout``, - ``base_url`` -``ngwmn`` ``ngwmn.NgwmnConfiguration`` the same five + ``base_url``, ``api_version`` +``ngwmn`` ``ngwmn.NgwmnConfiguration`` the same, without ``api_version`` ``nwdc`` ``nwdc.NwdcConfiguration`` ``concurrency``, ``retries``, ``stall_timeout``, ``base_url`` ``wqp``, ``nldi``, ``streamstats`` ``wqp.WqpConfiguration`` and so on ``retries``, ``stall_timeout``, @@ -547,6 +553,50 @@ host you gave a credential to. If the mirror needs its own credential, it needs its own mechanism. +.. _configuration-api-version: + +Pinning the Water Data API version +---------------------------------- + +The Water Data OGC getters request v1 of the Water Data APIs and shape +responses for it. ``api_version`` sends an adapter's requests to another +version, either within a ``configure`` block or, from the file, for every +script that reads it: + +.. code-block:: python + + with dataretrieval.configure(WaterdataConfiguration(api_version="v0")): + df, md = waterdata.get_time_series_metadata( + monitoring_location_id="USGS-05114000" + ) + +.. code-block:: toml + + [waterdata] + api_version = "v0" + +The value is the version segment of the service's path, ``"v1"`` or ``"v0"``, +and it replaces only that segment. The Samples database, the statistics service +and the STAC catalog are versioned separately and have no v1, so a pin leaves +them unchanged. v0 stays online until June 2027, after which the service +redirects every v0 request to v1. + +Under a pinned version, getters return that version's columns as the service +sends them. +v0 of ``time-series-metadata`` still has ``begin_utc``, ``end_utc``, +``state_name`` and ``hydrologic_unit_code``, which v1 removed, and v0 of +``field-measurements`` returns ``time`` as a datetime, where v1 returns a date +(parsed to a tz-naive midnight timestamp). +When a call to ``get_time_series_metadata`` names one of those four filters, +the getter sends that one call to v0 and emits a ``DeprecationWarning``. +It does not change your configuration, +so every other getter still uses the version you set. + +Unlike ``base_url``, the file accepts ``api_version``, +because a version cannot send a request to another host. +The environment refuses it, as it refuses every per-adapter setting. + + .. _configuration-secret-store: Keeping a key out of your environment entirely diff --git a/tests/configuration_test.py b/tests/configuration_test.py index e4045b79..01abc7f4 100644 --- a/tests/configuration_test.py +++ b/tests/configuration_test.py @@ -27,14 +27,15 @@ from dataretrieval.waterdata import WaterdataConfiguration from dataretrieval.wqp import WqpConfiguration -WATERDATA_URL = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" +_WATERDATA_HOST = "https://api.waterdata.usgs.gov" +WATERDATA_URL = f"{_WATERDATA_HOST}/ogcapi/v1/collections/daily/items" # Where the base-URL tests redirect to. A host the suite cannot connect to, so a # redirect that failed to apply shows up as an unmocked request rather than as # a real one. _MIRROR = "https://mirror.example/waterdata" _MIRROR_RE = re.compile(r"^https://mirror\.example/") -_WATERDATA_RE = re.compile(r"^https://api\.waterdata\.usgs\.gov/") +_WATERDATA_RE = re.compile(rf"^{re.escape(_WATERDATA_HOST)}/") # One committed page of the ``daily`` collection, shared with the Water Data suite. Real # response shape rather than a hand-made stub, so a redirect is exercised through the @@ -400,6 +401,7 @@ def _resolved_settings() -> dict[object, object]: adapter=adapter ) snapshot[(adapter, "base_url")] = configuration.base_url(adapter=adapter) + snapshot[(adapter, "api_version")] = configuration.api_version(adapter=adapter) return snapshot @@ -1409,14 +1411,25 @@ def test_base_url_is_refused_from_the_environment(monkeypatch): assert "only be set in code" in out.getvalue() -def test_a_code_base_url_redirects_every_water_data_endpoint_family(httpx_mock): - """One Water Data configuration redirects every endpoint family together.""" +@pytest.mark.parametrize( + ("setting", "root", "ogc_version"), + [ + ({"base_url": _MIRROR}, _MIRROR, "v1"), + ({"api_version": "v0"}, _WATERDATA_HOST, "v0"), + ], + ids=["base_url", "api_version"], +) +def test_base_url_changes_every_endpoint_family_and_api_version_only_ogc( + httpx_mock, setting, root, ogc_version +): + """``base_url`` redirects every endpoint family together; ``api_version`` + changes only the OGC family's version segment.""" httpx_mock.add_response(json=_DAILY_PAGE) httpx_mock.add_response(json={"data": []}) httpx_mock.add_response(json={"features": []}) httpx_mock.add_response(json={"features": []}) - with dataretrieval.configure(WaterdataConfiguration(base_url=_MIRROR)): + with dataretrieval.configure(WaterdataConfiguration(**setting)): waterdata.get_daily(monitoring_location_id="USGS-05427718") waterdata.get_codes("states") waterdata.get_stats_por( @@ -1431,11 +1444,71 @@ def test_a_code_base_url_redirects_every_water_data_endpoint_family(httpx_mock): ) requested = [str(request.url) for request in httpx_mock.get_requests()] - assert requested[0].startswith(f"{_MIRROR}/ogcapi/v0/collections/daily/items") - assert requested[1].startswith(f"{_MIRROR}/samples-data/codeservice/states") - assert requested[2].startswith(f"{_MIRROR}/statistics/v0/observationNormals") - assert requested[3].startswith(f"{_MIRROR}/stac/v0/search") - assert all(_WATERDATA_RE.match(url) is None for url in requested) + expected = [ + f"{root}/ogcapi/{ogc_version}/collections/daily/items", + f"{root}/samples-data/codeservice/states", + f"{root}/statistics/v0/observationNormals", + f"{root}/stac/v0/search", + ] + for url, prefix in zip(requested, expected, strict=True): + assert url.startswith(prefix) + + +def test_api_version_applies_from_code_and_from_the_file(config_file): + """Unlike a base URL, a version may come from the file, and a block takes + precedence over it.""" + config_file('[waterdata]\napi_version = "v0"\n') + assert configuration.api_version(adapter="waterdata") == "v0" + # Adapter-only: no other adapter reads it, and it has no package-wide value. + assert configuration.api_version(adapter="ngwmn") is None + + with dataretrieval.configure(WaterdataConfiguration(api_version="v1")): + assert configuration.api_version(adapter="waterdata") == "v1" + assert configuration.api_version(adapter="waterdata") == "v0" + + +def test_api_version_at_the_top_level_names_the_table_to_move_it_to(config_file): + """The error names the table to move the line to, not the top-level + settings.""" + config_file('api_version = "v0"\n') + with pytest.raises( + configuration.ConfigurationError, match=r"such as \[waterdata\]" + ): + configuration.api_version(adapter="waterdata") + + +def test_api_version_in_a_table_that_does_not_read_it_raises(config_file): + """A known setting in the wrong table is an error, not a warning.""" + config_file('[wqp]\napi_version = "v0"\n') + with pytest.raises( + configuration.ConfigurationError, match="not a setting that table accepts" + ): + configuration.retries(adapter="wqp") + + +def test_api_version_must_be_a_version_path_segment(): + """Checked as a shape, not against a list of known versions.""" + for bad in ("1", "V1", "v1.2", "latest"): + with pytest.raises(configuration.ConfigurationError, match="such as 'v1'"): + WaterdataConfiguration(api_version=bad) + assert WaterdataConfiguration(api_version="v2").api_version == "v2" + + +def test_api_version_is_refused_from_the_environment(monkeypatch): + """Refused, as every adapter-only setting is. The message also points to + the file, which accepts it.""" + monkeypatch.setenv("API_USGS_API_VERSION", "v0") + with pytest.raises(configuration.ConfigurationError, match="table of the file"): + configuration.api_version(adapter="waterdata") + + +def test_show_configuration_reports_a_pinned_api_version(config_file): + config_file('[waterdata]\napi_version = "v0"\n') + out = io.StringIO() + dataretrieval.show_configuration(stream=out) + override_section = out.getvalue().split("adapter overrides", 1)[1] + assert "api_version" in override_section + assert "v0" in override_section def test_a_code_base_url_redirects_the_adapters_requests(httpx_mock): @@ -1458,7 +1531,7 @@ def test_a_code_base_url_redirects_the_adapters_requests(httpx_mock): waterdata.get_daily(monitoring_location_id="USGS-05427718") direct_url = str(httpx_mock.get_requests()[-1].url) - assert redirected_url.startswith(f"{_MIRROR}/ogcapi/v0/collections/daily/items") + assert redirected_url.startswith(f"{_MIRROR}/ogcapi/v1/collections/daily/items") assert direct_url.startswith(WATERDATA_URL) streamstats_mirror = "https://mirror.example/streamstats" diff --git a/tests/contracts/README.md b/tests/contracts/README.md index c985f9da..c8d24e1d 100644 --- a/tests/contracts/README.md +++ b/tests/contracts/README.md @@ -9,7 +9,8 @@ The suite uses four dependency-oriented layers without moving established tests: `wqp_test.py`, `nldi_test.py`, `streamstats_test.py`): service request construction, response parsing, and documented protocol behavior. - **Component** (`transport_test.py`, `waterdata_chunking_test.py`, - `waterdata_queryables_test.py`, `rdb_test.py`, `_csv_test.py`): one internal + `waterdata_queryables_test.py`, `waterdata_endpoints_test.py`, + `waterdata_properties_test.py`, `rdb_test.py`, `_csv_test.py`): one internal responsibility in isolation. - **Cross-component** (`architecture_test.py`, `headers_host_scoping_test.py`, `waterdata_progress_test.py`): dependency fitness functions and behavior that diff --git a/tests/data/waterdata_ogc_fixtures.json b/tests/data/waterdata_ogc_fixtures.json index 0820e877..f53a53f4 100644 --- a/tests/data/waterdata_ogc_fixtures.json +++ b/tests/data/waterdata_ogc_fixtures.json @@ -265,18 +265,19 @@ ], "type": "Point" }, - "id": "1f6dacef-9405-4e72-a755-6d3ff6121051", + "id": "c7ab17e0-14cb-4998-b532-95353f75e6ee", "properties": { "approval_status": "Approved", - "last_modified": "2025-08-28T11:10:36.529563+00:00", + "last_modified": "2026-01-08T21:21:09.233391+00:00", + "method_category": "UNKWN", "monitoring_location_id": "USGS-06904500", "parameter_code": "00065", "qualifier": null, "statistic_id": "00011", - "time": "2025-01-01T00:00:00+00:00", + "time": "2025-09-22T16:00:00+00:00", "time_series_id": "b36569a0067443ac9425e850a3ac7baa", "unit_of_measure": "ft", - "value": "1.76" + "value": "0.39" }, "type": "Feature" }, @@ -288,18 +289,19 @@ ], "type": "Point" }, - "id": "0be08516-f3a0-4a60-b3bf-41fba6b0a3ea", + "id": "f76707d5-2a10-4de0-99c3-65b63b4a92db", "properties": { "approval_status": "Approved", - "last_modified": "2025-08-28T11:10:36.529563+00:00", + "last_modified": "2026-01-08T21:21:09.233391+00:00", + "method_category": "UNKWN", "monitoring_location_id": "USGS-06904500", "parameter_code": "00065", "qualifier": null, "statistic_id": "00011", - "time": "2025-01-01T00:15:00+00:00", + "time": "2025-09-22T16:15:00+00:00", "time_series_id": "b36569a0067443ac9425e850a3ac7baa", "unit_of_measure": "ft", - "value": "1.76" + "value": "0.39" }, "type": "Feature" } @@ -938,34 +940,30 @@ { "geometry": { "coordinates": [ - -89.35249999999999, - 43.20888888888889 + -74.5836111111111, + 40.9177777777778 ], "type": "Point" }, - "id": "04585e9fb4ec467aafa86c7c0b3f1439", + "id": "000068b2a47b45708c656bcff5264f54", "properties": { - "begin": "2013-10-01T06:00:00.000001", - "begin_utc": "2013-10-01T11:00:00+00:00", - "computation_identifier": "Decumulated", - "computation_period_identifier": "Points", - "data_gap_interval": "PT1H12M", - "end": "2025-10-01T05:45:00.000001", - "end_utc": "2025-10-01T10:45:00+00:00", - "hydrologic_unit_code": "070900020504", - "id": "04585e9fb4ec467aafa86c7c0b3f1439", - "last_modified": "2026-06-05T17:16:13.005607", - "monitoring_location_id": "USGS-05427718", - "parameter_code": "00045", - "parameter_description": "Precipitation, total, inches", - "parameter_name": "Precipitation", + "begin": "1983-11-03T05:00:00+00:00", + "computation_identifier": "Min", + "computation_period_identifier": "Daily", + "data_gap_interval": null, + "end": "1985-04-16T05:00:00+00:00", + "id": "000068b2a47b45708c656bcff5264f54", + "last_modified": "2017-05-07T20:53:29.551297", + "monitoring_location_id": "USGS-01379790", + "parameter_code": "00010", + "parameter_description": "Temperature, water, degrees Celsius", + "parameter_name": "Temperature, water", "parent_time_series_id": null, "primary": "Primary", - "state_name": "Wisconsin", - "statistic_id": null, + "statistic_id": "00002", "sublocation_identifier": null, "thresholds": [], - "unit_of_measure": "in", + "unit_of_measure": "degC", "web_description": null }, "type": "Feature" @@ -973,34 +971,30 @@ { "geometry": { "coordinates": [ - -89.35249999999999, - 43.20888888888889 + -112.13859444444445, + 40.780852777777774 ], "type": "Point" }, - "id": "0aa968f7d6494e2eb214d1829754532d", + "id": "00009b3da7e44a96a283bc413ff2de48", "properties": { - "begin": "1990-10-01T00:00:00.000001", - "begin_utc": "1990-10-01T05:00:00+00:00", - "computation_identifier": "Mean", - "computation_period_identifier": "Daily", - "data_gap_interval": null, - "end": "2025-09-30T00:00:00.000001", - "end_utc": "2025-09-30T05:00:00+00:00", - "hydrologic_unit_code": "070900020504", - "id": "0aa968f7d6494e2eb214d1829754532d", - "last_modified": "2025-12-19T19:35:48.365292", - "monitoring_location_id": "USGS-05427718", - "parameter_code": "91060", - "parameter_description": "Orthophosphate, water, filtered, pounds per day", - "parameter_name": "Orthophosphate, diss", - "parent_time_series_id": null, + "begin": "2006-05-18T12:00:00+00:00", + "computation_identifier": "Instantaneous", + "computation_period_identifier": "Points", + "data_gap_interval": "PT1H12M", + "end": "2026-09-22T16:00:00+00:00", + "id": "00009b3da7e44a96a283bc413ff2de48", + "last_modified": "2026-09-22T10:10:05.077878", + "monitoring_location_id": "USGS-10172640", + "parameter_code": "00060", + "parameter_description": "Discharge, cubic feet per second", + "parameter_name": "Discharge", + "parent_time_series_id": "769d7201a8044b4ab5218bac4cd282e8", "primary": "Primary", - "state_name": "Wisconsin", - "statistic_id": "00003", + "statistic_id": "00011", "sublocation_identifier": null, "thresholds": [], - "unit_of_measure": "lbs/day", + "unit_of_measure": "ft^3/s", "web_description": null }, "type": "Feature" diff --git a/tests/data/waterdata_queryables.json b/tests/data/waterdata_queryables.json index a9d9862c..93bf9411 100644 --- a/tests/data/waterdata_queryables.json +++ b/tests/data/waterdata_queryables.json @@ -119,6 +119,7 @@ "hydrologic_unit_code", "id", "last_modified", + "method_category", "minor_civil_division_code", "monitoring_location_id", "monitoring_location_name", @@ -487,14 +488,11 @@ ], "time-series-metadata": [ "begin", - "begin_utc", "computation_identifier", "computation_period_identifier", "data_gap_interval", "end", - "end_utc", "geometry", - "hydrologic_unit_code", "id", "last_modified", "monitoring_location_id", @@ -503,8 +501,8 @@ "parameter_name", "parent_time_series_id", "primary", - "state_name", "statistic_id", + "statistics_begin", "sublocation_identifier", "thresholds", "unit_of_measure", diff --git a/tests/headers_host_scoping_test.py b/tests/headers_host_scoping_test.py index 515435b3..f5aeeb03 100644 --- a/tests/headers_host_scoping_test.py +++ b/tests/headers_host_scoping_test.py @@ -27,7 +27,7 @@ def _api_token(self, monkeypatch: pytest.MonkeyPatch) -> None: def test_key_included_for_waterdata_host(self): """The key is added when target URL matches api.waterdata.usgs.gov.""" - url = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" + url = "https://api.waterdata.usgs.gov/ogcapi/v1/collections/daily/items" headers = _default_headers(url) assert headers.get("X-Api-Key") == self.FAKE_TOKEN @@ -51,7 +51,7 @@ def test_key_excluded_for_rating_asset_host(self): def test_key_excluded_for_lookalike_host(self): """The key is not sent to a typosquatting/lookalike domain.""" - url = "https://api.waterdata.usgs.gov.evil.com/ogcapi/v0/daily/items" + url = "https://api.waterdata.usgs.gov.evil.com/ogcapi/v1/daily/items" headers = _default_headers(url) assert "X-Api-Key" not in headers @@ -65,7 +65,7 @@ def test_key_excluded_when_no_token_set( ) -> None: """No key header at all when API_USGS_PAT is not set.""" monkeypatch.delenv("API_USGS_PAT") - headers = _default_headers("https://api.waterdata.usgs.gov/ogcapi/v0/daily") + headers = _default_headers("https://api.waterdata.usgs.gov/ogcapi/v1/daily") assert "X-Api-Key" not in headers def test_non_auth_headers_always_present(self): @@ -86,7 +86,7 @@ def test_key_excluded_over_cleartext_on_the_authorized_host(self): because of a hostname an attacker chose to keep -- reachable via a redirect or a server-supplied ``http://`` next-page link. """ - headers = _default_headers("http://api.waterdata.usgs.gov/ogcapi/v0/daily") + headers = _default_headers("http://api.waterdata.usgs.gov/ogcapi/v1/daily") assert "X-Api-Key" not in headers def test_sync_transport_withholds_key_on_downgrade_to_cleartext(self): diff --git a/tests/waterdata_endpoints_test.py b/tests/waterdata_endpoints_test.py new file mode 100644 index 00000000..6d3e8959 --- /dev/null +++ b/tests/waterdata_endpoints_test.py @@ -0,0 +1,108 @@ +"""Live monitors for the API version each Water Data family serves. + +``waterdata/endpoints.py`` puts a version in the OGC, STAC and statistics +paths. An old version keeps responding after USGS publishes a new one, so no +other test fails when that happens. (Samples and NGWMN have no version segment.) + +Two conditions are checked, in separate tests for OGC and STAC and in one +test for statistics: + +- **A new version is available**: the family's default is not the version the + package requests. +- **The version the package requests stopped working**: it no longer returns + data. + +The OGC and STAC roots publish a ``self`` link naming their default version, +so the version is read from it. OGC cannot be probed, because it responds with +200 and an empty body for any version segment (checked 2026-09-22). Statistics +has no root document, so it is probed; a missing statistics version responds +with 404. +""" + +import re + +import httpx +import pytest + +from dataretrieval.waterdata import endpoints + +#: The families whose root document names the default version and that have a +#: ``/collections`` endpoint, with the function that builds the requested URL. +_FAMILIES = { + "ogcapi": endpoints.ogc_api_url, + "stac": endpoints.ratings_catalog_url, +} + +#: A version segment anywhere in a path: ``/v0``, ``/v12/``, ``/v1?f=json``. +_VERSION_RE = re.compile(r"/(v\d+)(?=[/?#]|$)") + +#: Fail a hung request before the scheduled job's own timeout does. +_TIMEOUT = 60 + + +def _split_version(url: str) -> tuple[str, str]: + """Split *url* into its unversioned root and its version segment.""" + match = _VERSION_RE.search(url) + assert match is not None, f"no version segment in {url!r}" + return url[: match.start()] + "/", match.group(1) + + +def _served_version(root: str) -> str: + """The version in *root*'s ``self`` link, e.g. ``.../ogcapi/v1?f=json``.""" + response = httpx.get(root, timeout=_TIMEOUT, follow_redirects=True) + response.raise_for_status() + links = response.json().get("links") or [] + self_links = [link["href"] for link in links if link.get("rel") == "self"] + assert self_links, f"{root} published no self link: {links}" + return _split_version(self_links[0])[1] + + +@pytest.mark.live +@pytest.mark.parametrize("family", sorted(_FAMILIES)) +def test_service_default_is_the_version_this_package_requests(family): + root, requested = _split_version(_FAMILIES[family]()) + served = _served_version(root) + + assert served == requested, ( + f"the {family} API now serves {served} by default; this package requests " + f"{requested}. Move the pin in waterdata/endpoints.py to {served}, after " + f"checking the {served} release notes for dropped or renamed fields." + ) + + +@pytest.mark.live +def test_statistics_has_published_no_version_beyond_the_one_we_request(): + """``/statistics/vN`` responds with 404 even when vN exists, so the probe + requests ``/statistics/vN/docs``.""" + url = endpoints.statistics_api_url() + root, current = _split_version(url) + following = f"v{int(current.removeprefix('v')) + 1}" + + assert httpx.get(f"{url}/docs", timeout=_TIMEOUT).status_code == 200, ( + f"the statistics service stopped serving {current}, which this package " + "requests; check what replaced it." + ) + + probe = httpx.get(f"{root}{following}/docs", timeout=_TIMEOUT) + assert probe.status_code == 404, ( + f"the statistics service now responds to {following}/docs with HTTP " + f"{probe.status_code}; check whether the package should move to " + f"{following}, and whether the service publishes a root document from " + "which the version can be read instead of probed." + ) + + +@pytest.mark.live +@pytest.mark.parametrize("family", sorted(_FAMILIES)) +def test_the_version_this_package_requests_still_returns_data(family): + """Checked on content, not status: OGC responds with 200 and an empty body + for a version that does not exist.""" + url = _FAMILIES[family]() + response = httpx.get(f"{url}/collections", timeout=_TIMEOUT) + response.raise_for_status() + assert response.content and response.json().get("collections"), ( + f"{url}/collections returned no collections, so the " + f"{_split_version(url)[1]} {family} API this package requests has stopped " + "serving. Move the pin in waterdata/endpoints.py to the version the " + "service now serves." + ) diff --git a/tests/waterdata_progress_test.py b/tests/waterdata_progress_test.py index 2286c81a..c509d320 100644 --- a/tests/waterdata_progress_test.py +++ b/tests/waterdata_progress_test.py @@ -44,7 +44,7 @@ def _run_walk_pages(*, geopd, req, client): # The Water Data host is the only one that accepts ``API_USGS_PAT``, and so the # only one where pointing the user at API-key registration is useful advice. -_KEYED_URL = "https://api.waterdata.usgs.gov/ogcapi/v0/" +_KEYED_URL = "https://api.waterdata.usgs.gov/ogcapi/v1/" @pytest.fixture(autouse=True) diff --git a/tests/waterdata_properties_test.py b/tests/waterdata_properties_test.py new file mode 100644 index 00000000..666d0629 --- /dev/null +++ b/tests/waterdata_properties_test.py @@ -0,0 +1,73 @@ +"""Live monitor: the columns a getter documents are the ones its collection has. + +Some getters list their returned columns in the ``properties`` docstring, under +"Available options are:". The list is hand-written, so it goes stale when USGS +adds or removes a field. It is checked here rather than generated, because a +generated list would make the docs build depend on the live service and would +not appear in ``help()``. +""" + +import inspect +import re + +import pytest + +from dataretrieval import waterdata +from dataretrieval.ogc.schema import _check_ogc_requests +from dataretrieval.waterdata.endpoints import ogc_api_url + +#: Getters that list their returned columns, by collection. Written out rather +#: than discovered, so a getter that loses its list fails instead of being +#: skipped. get_channel is left out: its list names the output column +#: channel_measurements_id where the schema has id. +_DOCUMENTED = { + "daily": waterdata.get_daily, + "continuous": waterdata.get_continuous, + "latest-continuous": waterdata.get_latest_continuous, + "latest-daily": waterdata.get_latest_daily, + "monitoring-locations": waterdata.get_monitoring_locations, + "time-series-metadata": waterdata.get_time_series_metadata, +} + +#: The label may be split across two lines. The list ends at the next unindented +#: line of the dedented docstring, which is the next numpydoc parameter. +_COLUMNS_RE = re.compile(r"Available\s+options\s+are:(.*?)(?=\n\S|\Z)", re.S) + +#: Some schemas list ``id`` and some do not, but every getter accepts it, so it +#: is excluded from both sides. +_ALWAYS_REQUESTABLE = {"id"} + + +def _documented_properties(getter) -> set[str]: + """The column names *getter* lists in its ``properties`` docstring.""" + match = _COLUMNS_RE.search(inspect.getdoc(getter) or "") + assert match, f"{getter.__name__} no longer documents its columns" + return {n.strip().rstrip(".") for n in match.group(1).split(",") if n.strip()} + + +def _schema_properties(collection: str) -> set[str]: + """The columns *collection* publishes in its OGC schema document.""" + body, _ = _check_ogc_requests(collection, "schema", base_url=ogc_api_url()) + properties = body.get("properties") + assert properties, f"{collection} published no schema properties" + return set(properties) + + +@pytest.mark.live +@pytest.mark.parametrize("collection", sorted(_DOCUMENTED)) +def test_documented_columns_match_the_collection_schema(collection): + """A getter's documented column list matches what the collection publishes. + + For an added field, consider a named parameter too; a removed field is a + breaking change and belongs in NEWS. + """ + getter = _DOCUMENTED[collection] + documented = _documented_properties(getter) - _ALWAYS_REQUESTABLE + published = _schema_properties(collection) - _ALWAYS_REQUESTABLE + + assert documented == published, ( + f"{getter.__name__} documents the wrong columns for {collection}: " + f"missing={sorted(published - documented)}, " + f"stale={sorted(documented - published)}. Edit the 'Available options " + "are:' list in its properties docstring." + ) diff --git a/tests/waterdata_queryables_test.py b/tests/waterdata_queryables_test.py index a0b2c262..1464cd0e 100644 --- a/tests/waterdata_queryables_test.py +++ b/tests/waterdata_queryables_test.py @@ -11,7 +11,7 @@ import httpx, json from typing import get_args from dataretrieval.waterdata.types import WATERDATA_SERVICES - base = "https://api.waterdata.usgs.gov/ogcapi/v0" + base = "https://api.waterdata.usgs.gov/ogcapi/v1" snap = {} for c in get_args(WATERDATA_SERVICES): r = httpx.get(f"{base}/collections/{c}/queryables", timeout=30) @@ -36,7 +36,7 @@ # The OGC queryables endpoint for any Water Data collection. QUERYABLES_RE = re.compile( - r"^https://api\.waterdata\.usgs\.gov/ogcapi/v0/collections/[^/]+/queryables$" + r"^https://api\.waterdata\.usgs\.gov/ogcapi/v1/collections/[^/]+/queryables$" ) # A minimal queryables document (the JSON Schema shape the real endpoint returns). @@ -99,10 +99,10 @@ def test_get_queryables_unknown_collection_raises(httpx_mock): # --- passthrough queryables (mocked) --------------------------------------- _DAILY_ITEMS_RE = re.compile( - r"^https://api\.waterdata\.usgs\.gov/ogcapi/v0/collections/daily/items" + r"^https://api\.waterdata\.usgs\.gov/ogcapi/v1/collections/daily/items" ) _DAILY_SCHEMA_RE = re.compile( - r"^https://api\.waterdata\.usgs\.gov/ogcapi/v0/collections/daily/schema$" + r"^https://api\.waterdata\.usgs\.gov/ogcapi/v1/collections/daily/schema$" ) _EMPTY_FEATURES = { "type": "FeatureCollection", diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 60c25e79..91e9b500 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -13,6 +13,8 @@ import pytest from pandas import DataFrame +import dataretrieval +from dataretrieval import configuration from dataretrieval.ogc.requests import ( _check_monitoring_location_id, _normalize_str_iterable, @@ -24,6 +26,7 @@ _construct_cql_request as _construct_cql_request_explicit, ) from dataretrieval.waterdata import ( + WaterdataConfiguration, get_channel, get_combined_metadata, get_continuous, @@ -49,7 +52,11 @@ _get_args, ) -_OGC_BASE = "https://api.waterdata.usgs.gov/ogcapi/v0" +_OGC_BASE = "https://api.waterdata.usgs.gov/ogcapi/v1" +#: The version the deprecated time-series-metadata filters are served from. +#: Spelled out rather than derived from ``_OGC_BASE``, so that it still names +#: v0 once the default changes. +_V0_OGC_BASE = "https://api.waterdata.usgs.gov/ogcapi/v0" _STATS_BASE = "https://api.waterdata.usgs.gov/statistics/v0" #: Two real features per collection, captured from the live collection and trimmed. @@ -489,7 +496,7 @@ def test_construct_cql_request_post_verbatim_body(): assert req.method == "POST" assert req.headers["Content-Type"] == "application/query-cql-json" assert str(req.url).startswith( - "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" + "https://api.waterdata.usgs.gov/ogcapi/v1/collections/daily/items" ) # The body is sent through unchanged, not re-serialized. assert req.content.decode() == body @@ -646,11 +653,11 @@ def _schema_url(collection, *, base=_OGC_BASE): ) -def _mock_items(httpx_mock, collection, body=None, **kwargs): +def _mock_items(httpx_mock, collection, body=None, *, base=_OGC_BASE, **kwargs): """Serve ``collection``'s fixture for any ``/items`` request against it.""" httpx_mock.add_response( method=None, - url=_items_url(collection), + url=_items_url(collection, base=base), json=_fixture(collection) if body is None else body, **kwargs, ) @@ -898,6 +905,21 @@ def test_get_continuous(httpx_mock): assert "continuous_id" in df.columns assert df["time"].dtype.name.startswith("datetime64[") assert "UTC" in df["time"].dtype.name + # A code column stays the string the service sent. + assert df["method_category"].tolist() == ["UNKWN", "UNKWN"] + + +def test_get_continuous_sends_method_category(httpx_mock): + """The named ``method_category`` parameter is sent in the request as a filter.""" + _mock_items(httpx_mock, "continuous") + + get_continuous( + monitoring_location_id="USGS-06904500", + method_category=["STNRD", "LMTUS"], + ) + + qs = _sent(httpx_mock, "continuous")[0] + assert qs["method_category"] == ["STNRD,LMTUS"] def test_get_latest_continuous(httpx_mock): @@ -1156,6 +1178,134 @@ def test_get_time_series_metadata(httpx_mock): assert qs["bbox"] == ["-89.840355,42.853411,-88.818626,43.422598"] +def test_time_series_metadata_goes_to_v1_without_an_advisory(httpx_mock): + _mock_items(httpx_mock, "time-series-metadata") + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + get_time_series_metadata( + monitoring_location_id="USGS-05427718", begin="1990-01-01/.." + ) + + (sent,) = httpx_mock.get_requests() + assert str(sent.url).startswith(f"{_OGC_BASE}/collections/time-series-metadata") + assert "begin=1990-01-01" in str(sent.url) + + +@pytest.mark.parametrize( + ("kwargs", "spelled"), + [ + ({"begin_utc": "1990-01-01/.."}, "begin_utc"), + ({"end_utc": "../2020-01-01"}, "end_utc"), + ({"state": "WI"}, "state"), + ({"state_name": "Wisconsin"}, "state_name"), + ({"hydrologic_unit_code": "07090002"}, "hydrologic_unit_code"), + ({"properties": ["begin_utc", "id"]}, "begin_utc"), + ], +) +def test_time_series_metadata_v0_only_filters_warn_and_go_to_v0( + httpx_mock, kwargs, spelled +): + """A call naming a field that v1 dropped, as a filter or in ``properties``, + goes to v0, and the warning uses the caller's spelling.""" + _mock_items(httpx_mock, "time-series-metadata", base=_V0_OGC_BASE) + + with pytest.warns( + DeprecationWarning, + match=rf"'{spelled}' argument.*on or after 2027-06-01.*sent to v0", + ): + get_time_series_metadata(monitoring_location_id="USGS-05427718", **kwargs) + + (sent,) = httpx_mock.get_requests() + assert str(sent.url).startswith(f"{_V0_OGC_BASE}/collections/time-series-metadata") + + +def test_v0_routing_does_not_write_on_the_callers_configuration(httpx_mock): + """The getter sets v0 on the request, not in the configuration, so the + version the caller set still applies to every other call (ADR 0011).""" + _mock_items(httpx_mock, "time-series-metadata", base=_V0_OGC_BASE) + _mock_items(httpx_mock, "daily") + + with dataretrieval.configure(WaterdataConfiguration(api_version="v1")): + with pytest.warns(DeprecationWarning): + get_time_series_metadata( + monitoring_location_id="USGS-05427718", begin_utc="1990-01-01/.." + ) + # The caller's setting is unchanged. + get_daily(monitoring_location_id="USGS-05427718") + assert configuration.api_version(adapter="waterdata") == "v1" + + legacy, other = (str(r.url) for r in httpx_mock.get_requests()) + assert legacy.startswith(f"{_V0_OGC_BASE}/collections/time-series-metadata") + assert other.startswith(f"{_OGC_BASE}/collections/daily") + + +#: Named parameters for returned columns, by getter and collection, with a value +#: to send. +_NEWLY_NAMED = { + (get_channel, "channel-measurements"): { + "channel_location_direction": "left bank", + }, + (get_field_measurements, "field-measurements"): { + "control_condition": "Clear", + "day": 5, + "field_measurements_series_id": "abc123", + "measurement_rated": "Good", + "month": 3, + "reading_type": "Discharge", + "time_of_day": "17:30:00", + "year": 2020, + }, + (get_peaks, "peaks"): { + "qualifier": "Bd", + "time_of_day": "17:30:00", + "value": "847000", + }, + (get_combined_metadata, "combined-metadata"): { + "data_gap_interval": "P1D", + "reading_type": "Discharge", + }, + (get_monitoring_locations, "monitoring-locations"): { + "revision_created": "2024-01-01", + "revision_modified": "2024-01-01", + "revision_note": "corrected", + }, + (get_time_series_metadata, "time-series-metadata"): { + "data_gap_interval": "P1D", + "parameter_description": "Discharge", + "statistics_begin": "2000", + }, + (get_field_measurements_metadata, "field-measurements-metadata"): { + "reading_type": "Discharge", + }, +} + + +@pytest.mark.parametrize( + ("getter", "collection", "parameter", "value"), + [ + pytest.param( + getter, collection, parameter, value, id=f"{getter.__name__}-{parameter}" + ) + for (getter, collection), params in _NEWLY_NAMED.items() + for parameter, value in params.items() + ], +) +def test_newly_named_columns_reach_the_request( + httpx_mock, getter, collection, parameter, value +): + """A newly named parameter is sent to the service. + + A call whose parameter is accepted but not forwarded returns unfiltered + results instead of failing. + """ + _mock_items(httpx_mock, collection) + + getter(monitoring_location_id="USGS-05427718", **{parameter: value}) + + assert _sent(httpx_mock, collection)[0][parameter] == [str(value)] + + def test_get_combined_metadata(httpx_mock): _mock_items(httpx_mock, "combined-metadata") diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index f0d6eaf2..9eaf3176 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1153,11 +1153,11 @@ def test_403_without_an_envelope_names_the_credential_cause(): def test_error_messages_name_the_url(): """Without the URL a failed chunk in a fan-out cannot be traced back to the request that produced it -- the message is all the interruption holds.""" - request = httpx.Request("GET", "https://api.waterdata.usgs.gov/ogcapi/v0/x") + request = httpx.Request("GET", "https://api.waterdata.usgs.gov/ogcapi/v1/x") resp = httpx.Response(400, content=b"", request=request) with pytest.raises(HTTPError) as excinfo: _raise_for_non_200(resp) - assert "https://api.waterdata.usgs.gov/ogcapi/v0/x" in str(excinfo.value) + assert "https://api.waterdata.usgs.gov/ogcapi/v1/x" in str(excinfo.value) def test_error_message_survives_a_response_with_no_request():