From c3feaca3782314e7bd59f8206c0690fe80e8adae Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Tue, 22 Sep 2026 21:29:42 -0500 Subject: [PATCH] fix(waterdata): accept the continuous method_category queryable USGS added method_category to the continuous collection in September 2026, which failed the nightly queryables snapshot test. It is the RLMS method category code (STNRD, LMTUS, EXPER, UNKWN). It is returned on every record and is null for time series that have not been categorized. latest-continuous does not have it. - Regenerate waterdata_queryables.json and the continuous fixture. - Add method_category to get_continuous's signature, its docstring, and its list of returned columns. - Add live tests for the API version each Water Data family serves. The OGC and STAC versions are read from the self link in each root document, because OGC answers 200 for any version segment. Statistics has no root document, so the next version is probed. A second test checks that the version the package requests still returns data. Co-Authored-By: Claude Opus 5.5 (1M context) --- NEWS.md | 2 + dataretrieval/waterdata/time_series.py | 10 +- tests/contracts/README.md | 4 +- tests/data/waterdata_ogc_fixtures.json | 18 ++-- tests/data/waterdata_queryables.json | 1 + tests/waterdata_endpoints_test.py | 136 +++++++++++++++++++++++++ tests/waterdata_test.py | 15 +++ 7 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 tests/waterdata_endpoints_test.py diff --git a/NEWS.md b/NEWS.md index f4dbd0331..a0cfb0318 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**09/22/2026:** `waterdata.get_continuous()` accepts `method_category`, a column the `continuous` collection added in September 2026: the RLMS method category code (`STNRD`, `LMTUS`, `EXPER` or `UNKWN`) for the method in effect over an observation's interval. It is returned on every record and is null for time series that have not been categorized. It could already be passed through `**queryables`; it is now a documented parameter. `get_latest_continuous()` is unchanged, because `latest-continuous` does not have the field. + **09/09/2026:** **Bug fix:** code and identifier columns keep their leading zeros. A bare `pandas.read_csv` infers a zero-padded code as a number, so `waterdata.get_samples()` returned parameter code `00060` as `60` and HUC12 `070700050502` as `70700050502`, and `nwis.get_info()` returned `huc_cd` `02060005` as `2060005`. One rule now decides what a code column is — a name ending in `code`, the RDB abbreviation `_cd`, or a name containing `identifier`, `huc`, or `fips` — and every delimited response is parsed through it: the Samples and WQP CSV readers, `rdb.read_rdb` (which reads the names from the RDB header rather than the caller listing them), and the Water Use CSV pages. **Behavior change:** these columns now hold strings. `waterdata.get_samples()`: `USGSpcode`, `Location_HUCEightDigitCode`, `Location_HUCTwelveDigitCode`, `SampleCollectionMethod_Identifier` (`get_samples_summary()` shares the parse; no column in its current profile was affected). `nwis.get_info()`, `nwis.what_sites()`, and `nwis.get_record(service="site")`: `huc_cd`, `state_cd`, `county_cd`, `district_cd`. A comparison against a number — `df["USGSpcode"] == 60` — or a merge onto a numeric key now matches nothing instead of raising, so compare against the padded string (`== "00060"`) or call `.astype(int)` where the number is what you want. **Behavior change:** a count whose name reads as an identifier is numeric again. WQP's `AlternateLocation_IdentifierCount` has been read as text since 05/31/2026 because "Identifier" appears in its name; a name ending in `count` is now excluded from the rule, so the same column has one dtype in every service that reports it. Measurement columns are unchanged, and the `waterdata` OGC getters and `ngwmn` were never affected: their JSON responses deliver codes as strings and numeric coercion there is limited to a fixed list of measurement columns. **Correction to the 1.2.0 notes:** the same fix was applied to the nine `wqp` getters on 05/31/2026 and never recorded here — `wqp.get_results()` and the `what_*` getters have returned HUCs, parameter codes, and FIPS codes as strings since that release. **09/01/2026:** **Announcement:** We at USGS Water Data for the Nation want your feedback! Tell us how we're doing by taking our quick [survey](https://usgswaterresources.gov1.qualtrics.com/jfe/form/SV_07gX8G1DeOtVrH8), available through September 2026. diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index c83232f0e..373c4d31e 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -267,6 +267,7 @@ def get_continuous( approval_status: str | Iterable[str] | None = None, unit_of_measure: str | Iterable[str] | None = None, qualifier: str | Iterable[str] | None = None, + method_category: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, last_modified: str | Iterable[str] | None = None, time: str | Iterable[str] | None = None, @@ -316,7 +317,8 @@ def get_continuous( The columns to return from the query. Available options are: geometry, id, time_series_id, monitoring_location_id, parameter_code, statistic_id, time, value, - unit_of_measure, approval_status, qualifier, last_modified + unit_of_measure, approval_status, qualifier, method_category, + last_modified time_series_id : string or iterable of strings, optional A unique identifier representing a single time series, corresponding to the id field in the time-series-metadata endpoint. @@ -345,6 +347,12 @@ def get_continuous( qualifier : string or iterable of strings, optional Any qualifiers associated with an observation, for instance whether a sensor may have been impacted by ice or whether values were estimated. + method_category : string or iterable of strings, optional + The RLMS method category code for the method in effect over the + observation's interval: "STNRD" (standardized, with known uncertainty + and full QA/QC), "LMTUS" (limited use: a modified or externally + sourced method), "EXPER" (experimental), or "UNKWN" (uncategorized). + Null for time series that have not been categorized. value : string or iterable of strings, optional The value of the observation. Values are transmitted as strings in the JSON response format to preserve precision. diff --git a/tests/contracts/README.md b/tests/contracts/README.md index c985f9daa..3599c99cd 100644 --- a/tests/contracts/README.md +++ b/tests/contracts/README.md @@ -9,8 +9,8 @@ The suite uses four dependency-oriented layers without moving established tests: `wqp_test.py`, `nldi_test.py`, `streamstats_test.py`): service request construction, response parsing, and documented protocol behavior. - **Component** (`transport_test.py`, `waterdata_chunking_test.py`, - `waterdata_queryables_test.py`, `rdb_test.py`, `_csv_test.py`): one internal - responsibility in isolation. + `waterdata_queryables_test.py`, `waterdata_endpoints_test.py`, `rdb_test.py`, + `_csv_test.py`): one internal responsibility in isolation. - **Cross-component** (`architecture_test.py`, `headers_host_scoping_test.py`, `waterdata_progress_test.py`): dependency fitness functions and behavior that spans adapters, OGC, transport, or security boundaries. diff --git a/tests/data/waterdata_ogc_fixtures.json b/tests/data/waterdata_ogc_fixtures.json index 0820e8776..93cb3ca09 100644 --- a/tests/data/waterdata_ogc_fixtures.json +++ b/tests/data/waterdata_ogc_fixtures.json @@ -265,18 +265,19 @@ ], "type": "Point" }, - "id": "1f6dacef-9405-4e72-a755-6d3ff6121051", + "id": "c7ab17e0-14cb-4998-b532-95353f75e6ee", "properties": { "approval_status": "Approved", - "last_modified": "2025-08-28T11:10:36.529563+00:00", + "last_modified": "2026-01-08T21:21:09.233391+00:00", + "method_category": "UNKWN", "monitoring_location_id": "USGS-06904500", "parameter_code": "00065", "qualifier": null, "statistic_id": "00011", - "time": "2025-01-01T00:00:00+00:00", + "time": "2025-09-22T16:00:00+00:00", "time_series_id": "b36569a0067443ac9425e850a3ac7baa", "unit_of_measure": "ft", - "value": "1.76" + "value": "0.39" }, "type": "Feature" }, @@ -288,18 +289,19 @@ ], "type": "Point" }, - "id": "0be08516-f3a0-4a60-b3bf-41fba6b0a3ea", + "id": "f76707d5-2a10-4de0-99c3-65b63b4a92db", "properties": { "approval_status": "Approved", - "last_modified": "2025-08-28T11:10:36.529563+00:00", + "last_modified": "2026-01-08T21:21:09.233391+00:00", + "method_category": "UNKWN", "monitoring_location_id": "USGS-06904500", "parameter_code": "00065", "qualifier": null, "statistic_id": "00011", - "time": "2025-01-01T00:15:00+00:00", + "time": "2025-09-22T16:15:00+00:00", "time_series_id": "b36569a0067443ac9425e850a3ac7baa", "unit_of_measure": "ft", - "value": "1.76" + "value": "0.39" }, "type": "Feature" } diff --git a/tests/data/waterdata_queryables.json b/tests/data/waterdata_queryables.json index a9d9862c1..6cee9a5c2 100644 --- a/tests/data/waterdata_queryables.json +++ b/tests/data/waterdata_queryables.json @@ -119,6 +119,7 @@ "hydrologic_unit_code", "id", "last_modified", + "method_category", "minor_civil_division_code", "monitoring_location_id", "monitoring_location_name", diff --git a/tests/waterdata_endpoints_test.py b/tests/waterdata_endpoints_test.py new file mode 100644 index 000000000..1f4a1b804 --- /dev/null +++ b/tests/waterdata_endpoints_test.py @@ -0,0 +1,136 @@ +"""Live monitors for the API version each Water Data family serves. + +``waterdata/endpoints.py`` puts a version in the OGC, STAC and statistics +paths. An old version keeps answering after USGS publishes a new one, so no +other test fails when that happens. (Samples and NGWMN have no version segment.) + +Two conditions are checked separately: + +- **A new version is available**: the family's default no longer matches + :data:`_DEFAULT_VERSIONS`. +- **The version the package requests stopped working**: it no longer returns + data. + +The OGC and STAC roots publish a ``self`` link naming their default version, +so the version is read from it. OGC cannot be probed, because it answers 200 +with an empty body for any version segment. Statistics has no root document, +so it is probed; a missing statistics version answers 404. +""" + +import re + +import httpx +import pytest + +from dataretrieval.waterdata import endpoints + +#: The version each family serves by default, read from the live service on +#: 2026-09-22. This records what the service serves, not what the package +#: requests. Update it only after the package has moved to the new version, so +#: the test keeps failing until then. +_DEFAULT_VERSIONS = { + "ogcapi": "v1", + "stac": "v0", +} + +#: The endpoint function that builds each family's URL, so the version checked +#: is the one the package actually requests. +_FAMILIES = { + "ogcapi": endpoints.ogc_api_url, + "stac": endpoints.ratings_catalog_url, + "statistics": endpoints.statistics_api_url, +} + +#: A version segment anywhere in a path: ``/v0``, ``/v12/``, ``/v1?f=json``. +_VERSION_RE = re.compile(r"/(v\d+)(?=[/?#]|$)") + +#: Fail a hung request before the scheduled job's own timeout does. +_TIMEOUT = 60 + + +def _split_version(url: str) -> tuple[str, str]: + """Split *url* into its unversioned root and its version segment.""" + match = _VERSION_RE.search(url) + assert match is not None, f"no version segment in {url!r}" + return url[: match.start()] + "/", match.group(1) + + +def _served_version(root: str) -> str: + """The version in *root*'s ``self`` link, e.g. ``.../ogcapi/v1?f=json``.""" + response = httpx.get(root, timeout=_TIMEOUT, follow_redirects=True) + response.raise_for_status() + links = response.json().get("links") or [] + self_links = [link["href"] for link in links if link.get("rel") == "self"] + assert self_links, f"{root} published no self link: {links}" + return _split_version(self_links[0])[1] + + +@pytest.mark.live +@pytest.mark.parametrize("family", sorted(_DEFAULT_VERSIONS)) +def test_service_still_serves_the_recorded_default_version(family): + """The version a family serves by default is the one recorded here. + + On failure, move the pin in ``waterdata/endpoints.py`` to the new version + (check its release notes for dropped or renamed fields), then update + ``_DEFAULT_VERSIONS``. + """ + root, _ = _split_version(_FAMILIES[family]()) + served = _served_version(root) + + assert served == _DEFAULT_VERSIONS[family], ( + f"the {family} API now serves {served} by default, not " + f"{_DEFAULT_VERSIONS[family]}. Move the package to {served} and update " + "_DEFAULT_VERSIONS." + ) + + +@pytest.mark.live +def test_statistics_has_published_no_version_beyond_the_one_we_request(): + """Statistics has no root document, so the next version is probed. + + Both the unversioned root and ``/statistics/vN`` answer 404, so the + ``/docs`` page is what shows whether a version exists. + """ + url = _FAMILIES["statistics"]() + root, current = _split_version(url) + following = f"v{int(current.removeprefix('v')) + 1}" + + assert httpx.get(f"{url}/docs", timeout=_TIMEOUT).status_code == 200, ( + f"the statistics service stopped serving {current}, which this package " + "requests; check what replaced it." + ) + + probe = httpx.get(f"{root}{following}/docs", timeout=_TIMEOUT) + assert probe.status_code == 404, ( + f"the statistics service now answers for {following} " + f"(HTTP {probe.status_code}); check whether the package should move to " + "it, and whether it publishes a root document that would let this be " + "discovered rather than probed." + ) + + +@pytest.mark.live +@pytest.mark.parametrize("family", sorted(_FAMILIES)) +def test_the_version_this_package_requests_still_returns_data(family): + """The version the package pins still returns data. + + Checked on content, not status: OGC answers 200 with an empty body for a + version that does not exist. + """ + url = _FAMILIES[family]() + + if family == "statistics": + # No collections endpoint; its docs page is what proves it is up. + response = httpx.get(f"{url}/docs", timeout=_TIMEOUT) + response.raise_for_status() + assert response.text.strip(), f"{url}/docs returned an empty body" + return + + response = httpx.get(f"{url}/collections", timeout=_TIMEOUT) + response.raise_for_status() + collections = response.json().get("collections") + assert collections, ( + f"{url}/collections returned no collections, so the " + f"{_split_version(url)[1]} {family} API this package requests has stopped " + "serving. Move the package to the version the service now offers." + ) diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 60c25e79d..b323cf4b1 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -898,6 +898,21 @@ def test_get_continuous(httpx_mock): assert "continuous_id" in df.columns assert df["time"].dtype.name.startswith("datetime64[") assert "UTC" in df["time"].dtype.name + # A code column stays the string the service sent. + assert df["method_category"].tolist() == ["UNKWN", "UNKWN"] + + +def test_get_continuous_sends_method_category(httpx_mock): + """The named ``method_category`` parameter reaches the request as a filter.""" + _mock_items(httpx_mock, "continuous") + + get_continuous( + monitoring_location_id="USGS-06904500", + method_category=["STNRD", "LMTUS"], + ) + + qs = _sent(httpx_mock, "continuous")[0] + assert qs["method_category"] == ["STNRD,LMTUS"] def test_get_latest_continuous(httpx_mock):