From 44d036e7cfde0f9f539a9b7a8042268c1034ad41 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 10 Aug 2026 09:07:00 +0000
Subject: [PATCH 01/44] Update CDC500 state query to use new TimeSeries and
 Observation tables

---
 scripts/us_cdc/cdc500_state/process.py | 99 +++++++++++++++-----------
 1 file changed, 58 insertions(+), 41 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 0115b96e43..ea9b2ecbe7 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -22,49 +22,66 @@
     os.mkdir(_OUTPUT_FILE_PATH)
 
 query = """
-SELECT distinct * from(
-SELECT 
-  statvar, 
-  SUBSTR(observation_about,0,8) as observation_about,
-  observation_date, 
-  CONCAT('dcAggregate/',measurement_method) as measurement_method,
-  population_statvar,
-  SUM(CAST(pop_count AS FLOAT64))*100/SUM(CAST(population AS FLOAT64)) as percent
-FROM 
-(
+WITH cdc_sv AS (
   SELECT
-    SVO1.variable_measured as statvar,
-    SVO1.observation_about as observation_about,
-    SVO1.observation_date as observation_date,
-    SVO1.value as percent,
-    SVO1.measurement_method as measurement_method,
-    SVO2.variable_measured as population_statvar,
-    SVO2.value as population,
-    CAST(SVO2.value AS FLOAT64) * CAST(SVO1.value AS FLOAT64) / 100 as pop_count
-  FROM `datcom-store.dc_kg_latest.StatVarObservation` as SVO1
-  JOIN `datcom-store.dc_kg_latest.StatVarObservation` as SVO2 ON TRUE
-  JOIN (
-    # Get the statvars and corresponding population statvar
-    # with ‘Percent_’ replaced with ‘Count_’ and
-    # dropping the non-age, non-gender constraints.
+    variable_measured AS cdc500,
+    CONCAT('Count_', REGEXP_SUBSTR(variable_measured, '(Person_.*ale|Person_.*Years|Person)')) AS pop_statvar
+  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries`
+  WHERE provenance = 'dc/base/CDC500'
+    AND variable_measured LIKE 'Percent_%'
+  GROUP BY cdc500, pop_statvar
+),
+svo_percent AS (
+  SELECT
+    O.variable_measured AS statvar,
+    O.entity1 AS observation_about,
+    O.date AS observation_date,
+    O.value AS percent,
+    T.measurement_method AS measurement_method,
+    cdc_sv.pop_statvar
+  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation` AS O
+  JOIN `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries` AS T
+    ON O.variable_measured = T.variable_measured
+    AND O.entity1 = T.entity1
+    AND O.facet_id = T.facet_id
+  JOIN cdc_sv ON O.variable_measured = cdc_sv.cdc500
+  WHERE O.entity1 LIKE 'geoId/%'
+),
+svo_count AS (
+  SELECT
+    variable_measured AS population_statvar,
+    entity1 AS observation_about,
+    date AS observation_date,
+    value AS population
+  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation`
+  WHERE entity1 LIKE 'geoId/%'
+    AND variable_measured IN (SELECT DISTINCT pop_statvar FROM cdc_sv)
+)
+SELECT DISTINCT * FROM (
+  SELECT 
+    statvar, 
+    SUBSTR(observation_about, 0, 8) AS observation_about,
+    observation_date, 
+    CONCAT('dcAggregate/', measurement_method) AS measurement_method,
+    population_statvar,
+    SUM(CAST(pop_count AS FLOAT64)) * 100 / SUM(CAST(population AS FLOAT64)) AS percent
+  FROM (
     SELECT
-      SVO.variable_measured as CDC500,
-      CONCAT('Count_', REGEXP_SUBSTR(SVO.variable_measured, '(Person_.*ale|Person_.*Years|Person)')) as pop_statvar
-      FROM `datcom-store.dc_kg_latest.StatVarObservation` as SVO
-    WHERE
-      SVO.prov_id = 'dc/base/CDC500'
-      AND SVO.variable_measured like 'Percent_%'
-      GROUP BY CDC500, pop_statvar
-   ) AS CDC_SV ON TRUE
-  WHERE
-    SVO1.prov_id = 'dc/base/CDC500'
-    AND SVO1.variable_measured LIKE 'Percent%'
-    AND SVO1.observation_about = SVO2.observation_about
-    AND SVO1.observation_date = SVO2.observation_date
-    AND SVO1.variable_measured = CDC_SV.CDC500
-    AND SVO2.variable_measured = CDC_SV.pop_statvar
-    AND SVO1.observation_about like "geoId/%"
-) group by 1,2,3,4,5
+      p.statvar,
+      p.observation_about,
+      p.observation_date,
+      p.percent,
+      p.measurement_method,
+      c.population_statvar,
+      c.population,
+      CAST(c.population AS FLOAT64) * CAST(p.percent AS FLOAT64) / 100 AS pop_count
+    FROM svo_percent AS p
+    JOIN svo_count AS c
+      ON p.observation_about = c.observation_about
+      AND p.observation_date = c.observation_date
+      AND p.pop_statvar = c.population_statvar
+  )
+  GROUP BY 1, 2, 3, 4, 5
 )
 """
 

From ebc79b75c1a48b9163aa7f0d88c16a43275e0609 Mon Sep 17 00:00:00 2001
From: shvngisingh <shvngisingh@google.com>
Date: Mon, 10 Aug 2026 17:16:40 +0530
Subject: [PATCH 02/44] Update scripts/us_cdc/cdc500_state/process.py

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 scripts/us_cdc/cdc500_state/process.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index ea9b2ecbe7..63c8e25bda 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -64,7 +64,7 @@
     observation_date, 
     CONCAT('dcAggregate/', measurement_method) AS measurement_method,
     population_statvar,
-    SUM(CAST(pop_count AS FLOAT64)) * 100 / SUM(CAST(population AS FLOAT64)) AS percent
+    SAFE_DIVIDE(SUM(CAST(pop_count AS FLOAT64)) * 100, SUM(CAST(population AS FLOAT64))) AS percent
   FROM (
     SELECT
       p.statvar,

From fdcdbb5378a334274919b364df6182b339de8903 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 27 Aug 2026 11:43:10 +0000
Subject: [PATCH 03/44] Address review comments: optimize joins, add provenance
 filters, and streamline aggregation in CDC500 state query

---
 scripts/us_cdc/cdc500_state/process.py | 71 +++++++++++++-------------
 1 file changed, 36 insertions(+), 35 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 63c8e25bda..36e1f11af6 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -31,6 +31,7 @@
     AND variable_measured LIKE 'Percent_%'
   GROUP BY cdc500, pop_statvar
 ),
+
 svo_percent AS (
   SELECT
     O.variable_measured AS statvar,
@@ -40,49 +41,49 @@
     T.measurement_method AS measurement_method,
     cdc_sv.pop_statvar
   FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation` AS O
-  JOIN `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries` AS T
+  INNER JOIN `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries` AS T
     ON O.variable_measured = T.variable_measured
     AND O.entity1 = T.entity1
     AND O.facet_id = T.facet_id
-  JOIN cdc_sv ON O.variable_measured = cdc_sv.cdc500
+    AND T.provenance = 'dc/base/CDC500'
+    AND T.variable_measured LIKE 'Percent_%'
+  INNER JOIN cdc_sv
+    ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
+    AND O.variable_measured LIKE 'Percent_%'
 ),
+
 svo_count AS (
   SELECT
-    variable_measured AS population_statvar,
-    entity1 AS observation_about,
-    date AS observation_date,
-    value AS population
-  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation`
-  WHERE entity1 LIKE 'geoId/%'
-    AND variable_measured IN (SELECT DISTINCT pop_statvar FROM cdc_sv)
-)
-SELECT DISTINCT * FROM (
-  SELECT 
-    statvar, 
-    SUBSTR(observation_about, 0, 8) AS observation_about,
-    observation_date, 
-    CONCAT('dcAggregate/', measurement_method) AS measurement_method,
-    population_statvar,
-    SAFE_DIVIDE(SUM(CAST(pop_count AS FLOAT64)) * 100, SUM(CAST(population AS FLOAT64))) AS percent
-  FROM (
-    SELECT
-      p.statvar,
-      p.observation_about,
-      p.observation_date,
-      p.percent,
-      p.measurement_method,
-      c.population_statvar,
-      c.population,
-      CAST(c.population AS FLOAT64) * CAST(p.percent AS FLOAT64) / 100 AS pop_count
-    FROM svo_percent AS p
-    JOIN svo_count AS c
-      ON p.observation_about = c.observation_about
-      AND p.observation_date = c.observation_date
-      AND p.pop_statvar = c.population_statvar
-  )
-  GROUP BY 1, 2, 3, 4, 5
+    O.variable_measured AS population_statvar,
+    O.entity1 AS observation_about,
+    O.date AS observation_date,
+    O.value AS population
+  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation` AS O
+  INNER JOIN (
+    SELECT DISTINCT pop_statvar
+    FROM cdc_sv
+  ) AS pop
+    ON O.variable_measured = pop.pop_statvar
+  WHERE O.entity1 LIKE 'geoId/%'
 )
+
+SELECT 
+  p.statvar, 
+  SUBSTR(p.observation_about, 0, 8) AS observation_about,
+  p.observation_date, 
+  CONCAT('dcAggregate/', p.measurement_method) AS measurement_method,
+  p.pop_statvar AS population_statvar,
+  SAFE_DIVIDE(
+    SUM(CAST(c.population AS FLOAT64) * CAST(p.percent AS FLOAT64) / 100) * 100,
+    SUM(CAST(c.population AS FLOAT64))
+  ) AS percent
+FROM svo_percent AS p
+INNER JOIN svo_count AS c
+  ON p.observation_about = c.observation_about
+  AND p.observation_date = c.observation_date
+  AND p.pop_statvar = c.population_statvar
+GROUP BY 1, 2, 3, 4, 5
 """
 
 client = bigquery.Client()

From d2d76104f7bccc60788edffee505d554a91ed1f4 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 31 Aug 2026 11:51:38 +0000
Subject: [PATCH 04/44] Address code review findings for CDC500 state
 aggregation: fix Census ACS 5Yr facet join, explicit cohort CASE mapping,
 modular entrypoint, and add unit test suite

---
 scripts/us_cdc/cdc500_state/README.md       |  4 +-
 scripts/us_cdc/cdc500_state/manifest.json   |  3 -
 scripts/us_cdc/cdc500_state/process.py      | 83 +++++++++++++++------
 scripts/us_cdc/cdc500_state/process_test.py | 66 ++++++++++++++++
 4 files changed, 129 insertions(+), 27 deletions(-)
 create mode 100644 scripts/us_cdc/cdc500_state/process_test.py

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 79f5abb00c..07ce31a391 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -12,7 +12,7 @@ Author: Padma Gundapaneni @padma-g
 ## About the Dataset
 
 ### Overview
-The state level data is aggragated from city level data coming from CDC500 import. 
+The state level data is aggregated from city level data coming from CDC500 import. 
 
 To get the data for this import run:
 ```bash
@@ -24,7 +24,7 @@ $ python3 process.py
 ### Artifacts
 
 #### Scripts
-[`process.py`](https://github.com/datacommonsorg/data/blob/master//scripts/us_cdc/cdc500_state/process.py)
+[`process.py`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/process.py)
 
 
 #### tMCFs
diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index eae459b1c6..1d3909a12f 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -21,9 +21,6 @@
                 "memory": 64,
                 "disk": 100
             },
-            "source_files": [
-                "CDC500State_Output/CDC500State_Output.csv"
-            ],
             "cron_schedule": "0 1 * * 1"
         }
     ]
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 36e1f11af6..6e5d17877d 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2026 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,21 +11,36 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Processes CDC 500 cities data into aggregated state-level health indicators."""
 
 import os
+from absl import app
+from absl import flags
 from absl import logging
 from google.cloud import bigquery
+import pandas as pd
 
+_FLAGS = flags.FLAGS
 _MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
-_OUTPUT_FILE_PATH = os.path.join(_MODULE_DIR + '/CDC500State_Output')
-if not os.path.exists(_OUTPUT_FILE_PATH):
-    os.mkdir(_OUTPUT_FILE_PATH)
+_DEFAULT_OUTPUT_DIR = os.path.join(_MODULE_DIR, 'CDC500State_Output')
 
-query = """
+flags.DEFINE_string('output_dir', _DEFAULT_OUTPUT_DIR,
+                    'Directory to write output CSV.')
+
+QUERY = """
 WITH cdc_sv AS (
   SELECT
     variable_measured AS cdc500,
-    CONCAT('Count_', REGEXP_SUBSTR(variable_measured, '(Person_.*ale|Person_.*Years|Person)')) AS pop_statvar
+    CASE
+      WHEN variable_measured LIKE '%Female_50To74Years%' OR variable_measured LIKE '%50To74Years_Female%' THEN 'Count_Person_Female_50To74Years'
+      WHEN variable_measured LIKE '%Female_21To65Years%' OR variable_measured LIKE '%21To65Years_Female%' THEN 'Count_Person_Female_21To65Years'
+      WHEN variable_measured LIKE '%Female_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Female%' THEN 'Count_Person_Female_65OrMoreYears'
+      WHEN variable_measured LIKE '%Male_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Male%' THEN 'Count_Person_Male_65OrMoreYears'
+      WHEN variable_measured LIKE '%65OrMoreYears%' THEN 'Count_Person_65OrMoreYears'
+      WHEN variable_measured LIKE '%18To64Years%' THEN 'Count_Person_18To64Years'
+      WHEN variable_measured LIKE '%18OrMoreYears%' THEN 'Count_Person_18OrMoreYears'
+      ELSE 'Count_Person'
+    END AS pop_statvar
   FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries`
   WHERE provenance = 'dc/base/CDC500'
     AND variable_measured LIKE 'Percent_%'
@@ -60,6 +75,11 @@
     O.date AS observation_date,
     O.value AS population
   FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation` AS O
+  INNER JOIN `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries` AS T
+    ON O.variable_measured = T.variable_measured
+    AND O.entity1 = T.entity1
+    AND O.facet_id = T.facet_id
+    AND T.provenance = 'dc/base/CensusACS5YearSurvey'
   INNER JOIN (
     SELECT DISTINCT pop_statvar
     FROM cdc_sv
@@ -70,12 +90,12 @@
 
 SELECT 
   p.statvar, 
-  SUBSTR(p.observation_about, 0, 8) AS observation_about,
+  SUBSTR(p.observation_about, 1, 8) AS observation_about,
   p.observation_date, 
   CONCAT('dcAggregate/', p.measurement_method) AS measurement_method,
   p.pop_statvar AS population_statvar,
   SAFE_DIVIDE(
-    SUM(CAST(c.population AS FLOAT64) * CAST(p.percent AS FLOAT64) / 100) * 100,
+    SUM(CAST(c.population AS FLOAT64) * CAST(p.percent AS FLOAT64)),
     SUM(CAST(c.population AS FLOAT64))
   ) AS percent
 FROM svo_percent AS p
@@ -86,18 +106,37 @@
 GROUP BY 1, 2, 3, 4, 5
 """
 
-client = bigquery.Client()
-try:
-    logging.info("Running the query")
-    query_job = client.query(query)
-except Exception as e:
-    logging.fatal(f"Error faced while running the query {e}")
-try:
-    logging.info("Converting to dataframe")
-    results = query_job.to_dataframe()
-except Exception as e:
-    logging.info(f"Error faced while fetching results: {e}")
+def get_query() -> str:
+    """Returns the SQL query string for CDC 500 state aggregation."""
+    return QUERY
+
+def run_process(client: bigquery.Client, output_file: str) -> pd.DataFrame:
+    """Executes the BigQuery query and writes the resulting DataFrame to output_file."""
+    logging.info("Running BigQuery aggregation query...")
+    try:
+        query_job = client.query(get_query())
+    except Exception as e:
+        logging.fatal("Failed to submit BigQuery query: %s", e)
+        raise
+
+    logging.info("Fetching query results into dataframe...")
+    try:
+        df = query_job.to_dataframe()
+    except Exception as e:
+        logging.fatal("Failed to fetch query results into dataframe: %s", e)
+        raise
+
+    output_dir = os.path.dirname(output_file)
+    os.makedirs(output_dir, exist_ok=True)
+    logging.info("Writing %d rows to %s", len(df), output_file)
+    df.to_csv(output_file, index=False)
+    return df
+
+def main(argv):
+    del argv  # Unused.
+    client = bigquery.Client()
+    output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
+    run_process(client, output_file)
 
-logging.info("Writing output to CSV")
-output_file = os.path.join(_OUTPUT_FILE_PATH + "/CDC500State_Output.csv")
-results.to_csv(output_file, index=False)
+if __name__ == '__main__':
+    app.run(main)
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
new file mode 100644
index 0000000000..d00cc29dec
--- /dev/null
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -0,0 +1,66 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for CDC 500 State aggregation script."""
+
+import os
+import tempfile
+import unittest
+from unittest import mock
+import pandas as pd
+
+from scripts.us_cdc.cdc500_state import process
+
+class CDC500StateProcessTest(unittest.TestCase):
+
+    def test_get_query(self):
+        query = process.get_query()
+        self.assertIn("spanner_dc_graph_prod_DEFAULT.TimeSeries", query)
+        self.assertIn("spanner_dc_graph_prod_DEFAULT.Observation", query)
+        self.assertIn("dc/base/CDC500", query)
+        self.assertIn("dc/base/CensusACS5YearSurvey", query)
+        self.assertIn("SAFE_DIVIDE", query)
+        self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
+
+    def test_run_process_success(self):
+        mock_client = mock.MagicMock()
+        sample_data = pd.DataFrame({
+            'statvar': ['Percent_Person_18OrMoreYears_WithAnyDisability'],
+            'observation_about': ['geoId/06'],
+            'observation_date': ['2022'],
+            'measurement_method': ['dcAggregate/CrudePrevalence'],
+            'population_statvar': ['Count_Person_18OrMoreYears'],
+            'percent': [29.6479]
+        })
+        mock_client.query.return_value.to_dataframe.return_value = sample_data
+        
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            result_df = process.run_process(mock_client, output_file)
+            
+            mock_client.query.assert_called_once()
+            self.assertTrue(os.path.exists(output_file))
+            saved_df = pd.read_csv(output_file)
+            self.assertEqual(len(saved_df), 1)
+            self.assertEqual(saved_df['observation_about'].iloc[0], 'geoId/06')
+
+    def test_run_process_query_error(self):
+        mock_client = mock.MagicMock()
+        mock_client.query.side_effect = Exception("BigQuery Access Denied")
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            with self.assertRaises(Exception):
+                process.run_process(mock_client, output_file)
+
+if __name__ == '__main__':
+    unittest.main()

From 3415550706c01dd2366e579e2e325d2736a57dca Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 31 Aug 2026 12:52:34 +0000
Subject: [PATCH 05/44] Add validation_config.json, update manifest, add tMCF
 units and guard output path

---
 scripts/us_cdc/cdc500_state/cdc500_state.tmcf |  2 +
 scripts/us_cdc/cdc500_state/manifest.json     |  3 +-
 scripts/us_cdc/cdc500_state/process.py        |  3 +-
 .../cdc500_state/validation_config.json       | 54 +++++++++++++++++++
 4 files changed, 60 insertions(+), 2 deletions(-)
 create mode 100644 scripts/us_cdc/cdc500_state/validation_config.json

diff --git a/scripts/us_cdc/cdc500_state/cdc500_state.tmcf b/scripts/us_cdc/cdc500_state/cdc500_state.tmcf
index 73a294f9cf..b7d6106b85 100644
--- a/scripts/us_cdc/cdc500_state/cdc500_state.tmcf
+++ b/scripts/us_cdc/cdc500_state/cdc500_state.tmcf
@@ -4,5 +4,7 @@ variableMeasured: C:CDC->statvar
 observationAbout: C:CDC->observation_about
 observationDate: C:CDC->observation_date
 value: C:CDC->percent
+unit: Percent
+scalingFactor: 100
 measurementMethod: C:CDC->measurement_method
 observationPeriod: "P1Y"
\ No newline at end of file
diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index 1d3909a12f..4a2ac714b8 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -21,7 +21,8 @@
                 "memory": 64,
                 "disk": 100
             },
-            "cron_schedule": "0 1 * * 1"
+            "cron_schedule": "0 1 * * 1",
+            "validation_config_file": "validation_config.json"
         }
     ]
 }
\ No newline at end of file
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 6e5d17877d..c1b2608b4c 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -127,7 +127,8 @@ def run_process(client: bigquery.Client, output_file: str) -> pd.DataFrame:
         raise
 
     output_dir = os.path.dirname(output_file)
-    os.makedirs(output_dir, exist_ok=True)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
     logging.info("Writing %d rows to %s", len(df), output_file)
     df.to_csv(output_file, index=False)
     return df
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
new file mode 100644
index 0000000000..ec7154e2fd
--- /dev/null
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -0,0 +1,54 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0
+            }
+        },
+        {
+            "rule_id": "check_missing_refs_count",
+            "description": "Checks that there are no missing entity references in lint report.",
+            "validator": "MISSING_REFS_COUNT",
+            "params": {
+                "threshold": 0
+            }
+        },
+        {
+            "rule_id": "check_lint_error_count",
+            "description": "Checks that there are no lint errors during MCF generation.",
+            "validator": "LINT_ERROR_COUNT",
+            "params": {
+                "threshold": 0
+            }
+        },
+        {
+            "rule_id": "check_max_value_percentage",
+            "description": "Checks that all percentage StatVars do not exceed 100%.",
+            "validator": "MAX_VALUE_CHECK",
+            "params": {
+                "maximum": 100
+            }
+        },
+        {
+            "rule_id": "check_min_value_percentage",
+            "description": "Checks that all percentage StatVars are not below 0%.",
+            "validator": "MIN_VALUE_CHECK",
+            "params": {
+                "minimum": 0
+            }
+        },
+        {
+            "rule_id": "check_num_places_state_count",
+            "description": "Checks that state-level observations cover all 50-52 US state entities.",
+            "validator": "NUM_PLACES_COUNT",
+            "params": {
+                "minimum": 50,
+                "maximum": 52
+            }
+        }
+    ]
+}

From dbc69620be7711e6437a2472c42e4bc3cbe0f1f1 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 1 Sep 2026 13:04:43 +0000
Subject: [PATCH 06/44] Address code review feedback on CDC500 state
 aggregation: update README docs, remove default validation rules, use
 logging.error, and clean up query helpers

---
 scripts/us_cdc/cdc500_state/README.md         | 39 +++++++++++++++----
 scripts/us_cdc/cdc500_state/process.py        | 13 ++-----
 scripts/us_cdc/cdc500_state/process_test.py   |  6 +--
 .../cdc500_state/validation_config.json       | 24 ------------
 4 files changed, 39 insertions(+), 43 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 07ce31a391..e651276c93 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -5,6 +5,8 @@ Author: Padma Gundapaneni @padma-g
 ## Table of Contents
 1. [About the Dataset](#about-the-dataset)
     1. [Overview](#overview)
+    2. [Data Sources and Tables](#data-sources-and-tables)
+    3. [Aggregation Methodology](#aggregation-methodology)
 2. [About the Import](#about-the-import)
     1. [Artifacts](#artifacts)
     2. [Import Procedure](#import-procedure)
@@ -12,12 +14,30 @@ Author: Padma Gundapaneni @padma-g
 ## About the Dataset
 
 ### Overview
-The state level data is aggregated from city level data coming from CDC500 import. 
+The state-level dataset calculates aggregated health indicator prevalence estimates for US states from the city-level CDC 500 Cities (`CDC500`) project data, weighted by corresponding Census ACS 5-Year population counts.
 
-To get the data for this import run:
-```bash
-$ python3 process.py
-```
+### Data Sources and Tables
+
+The aggregation script queries Google Cloud BigQuery graph tables in dataset `datcom-store.spanner_dc_graph_prod_DEFAULT`:
+
+1. **`TimeSeries`**:
+   - **CDC 500 Series**: Identifies CDC 500 Statistical Variables (`provenance = 'dc/base/CDC500'` and `variable_measured LIKE 'Percent_%'`) and extracts their measurement methods (`measurement_method`). It maps each percentage health metric to its appropriate denominator demographic cohort StatVar (e.g., `Count_Person_18OrMoreYears`, `Count_Person_Female_50To74Years`, `Count_Person_Female_21To65Years`, `Count_Person_65OrMoreYears`, etc.).
+   - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
+
+2. **`Observation`**:
+   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%'`) for CDC 500 StatVars.
+   - **City Cohort Populations**: Fetches city-level population counts (`value AS population`) for the corresponding demographic cohort StatVars.
+
+### Aggregation Methodology
+
+For each state, indicator StatVar, and observation date:
+- City observations are joined with their corresponding demographic population counts.
+- City geoIds (`geoId/XXXXXXX`) are mapped to state geoIds (`geoId/XX`) using the first 8 characters (including the prefix).
+- State-level prevalence percentages are computed as a population-weighted average:
+
+$$\text{State Percent} = \frac{\sum (\text{City Population} \times \text{City Percent})}{\sum \text{City Population}}$$
+
+The output measurement method is prefixed with `dcAggregate/` (e.g., `dcAggregate/CrudePrevalence`).
 
 ## About the Import
 
@@ -26,15 +46,20 @@ $ python3 process.py
 #### Scripts
 [`process.py`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/process.py)
 
+#### Unit Tests
+[`process_test.py`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/process_test.py)
 
-#### tMCFs
+#### tMCF Template
 [`cdc500_state.tmcf`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/cdc500_state.tmcf)
 
+#### Validation Config
+[`validation_config.json`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/validation_config.json)
+
 ### Import Procedure
 
 #### Data Download and Processing Steps
 
-To get the data for this import run:
+To run the BigQuery aggregation and generate the output CSV:
 
 ```bash
 $ python3 process.py
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index c1b2608b4c..e20cdf8a11 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -106,24 +106,20 @@
 GROUP BY 1, 2, 3, 4, 5
 """
 
-def get_query() -> str:
-    """Returns the SQL query string for CDC 500 state aggregation."""
-    return QUERY
-
-def run_process(client: bigquery.Client, output_file: str) -> pd.DataFrame:
+def run_process(client: bigquery.Client, output_file: str) -> None:
     """Executes the BigQuery query and writes the resulting DataFrame to output_file."""
     logging.info("Running BigQuery aggregation query...")
     try:
-        query_job = client.query(get_query())
+        query_job = client.query(QUERY)
     except Exception as e:
-        logging.fatal("Failed to submit BigQuery query: %s", e)
+        logging.error("Failed to submit BigQuery query: %s", e)
         raise
 
     logging.info("Fetching query results into dataframe...")
     try:
         df = query_job.to_dataframe()
     except Exception as e:
-        logging.fatal("Failed to fetch query results into dataframe: %s", e)
+        logging.error("Failed to fetch query results into dataframe: %s", e)
         raise
 
     output_dir = os.path.dirname(output_file)
@@ -131,7 +127,6 @@ def run_process(client: bigquery.Client, output_file: str) -> pd.DataFrame:
         os.makedirs(output_dir, exist_ok=True)
     logging.info("Writing %d rows to %s", len(df), output_file)
     df.to_csv(output_file, index=False)
-    return df
 
 def main(argv):
     del argv  # Unused.
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index d00cc29dec..eaec37e2f0 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -23,8 +23,8 @@
 
 class CDC500StateProcessTest(unittest.TestCase):
 
-    def test_get_query(self):
-        query = process.get_query()
+    def test_query_constants(self):
+        query = process.QUERY
         self.assertIn("spanner_dc_graph_prod_DEFAULT.TimeSeries", query)
         self.assertIn("spanner_dc_graph_prod_DEFAULT.Observation", query)
         self.assertIn("dc/base/CDC500", query)
@@ -46,7 +46,7 @@ def test_run_process_success(self):
         
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            result_df = process.run_process(mock_client, output_file)
+            process.run_process(mock_client, output_file)
             
             mock_client.query.assert_called_once()
             self.assertTrue(os.path.exists(output_file))
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index ec7154e2fd..5ac0fba24f 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -1,30 +1,6 @@
 {
     "schema_version": "1.0",
     "rules": [
-        {
-            "rule_id": "check_deleted_records_percent",
-            "description": "Checks that the percentage of deleted points is within the threshold.",
-            "validator": "DELETED_RECORDS_PERCENT",
-            "params": {
-                "threshold": 0
-            }
-        },
-        {
-            "rule_id": "check_missing_refs_count",
-            "description": "Checks that there are no missing entity references in lint report.",
-            "validator": "MISSING_REFS_COUNT",
-            "params": {
-                "threshold": 0
-            }
-        },
-        {
-            "rule_id": "check_lint_error_count",
-            "description": "Checks that there are no lint errors during MCF generation.",
-            "validator": "LINT_ERROR_COUNT",
-            "params": {
-                "threshold": 0
-            }
-        },
         {
             "rule_id": "check_max_value_percentage",
             "description": "Checks that all percentage StatVars do not exceed 100%.",

From 74dd551c3a9e44afa0529a8c68ea0009f213bfbe Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 2 Sep 2026 06:26:08 +0000
Subject: [PATCH 07/44] Address code review findings: remove unused pandas
 import, add dataframe error test, and fix senior cohort statvars

---
 scripts/us_cdc/cdc500_state/process.py      |  5 ++---
 scripts/us_cdc/cdc500_state/process_test.py | 10 ++++++++++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index e20cdf8a11..f4476810f3 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -18,7 +18,6 @@
 from absl import flags
 from absl import logging
 from google.cloud import bigquery
-import pandas as pd
 
 _FLAGS = flags.FLAGS
 _MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -34,8 +33,8 @@
     CASE
       WHEN variable_measured LIKE '%Female_50To74Years%' OR variable_measured LIKE '%50To74Years_Female%' THEN 'Count_Person_Female_50To74Years'
       WHEN variable_measured LIKE '%Female_21To65Years%' OR variable_measured LIKE '%21To65Years_Female%' THEN 'Count_Person_Female_21To65Years'
-      WHEN variable_measured LIKE '%Female_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Female%' THEN 'Count_Person_Female_65OrMoreYears'
-      WHEN variable_measured LIKE '%Male_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Male%' THEN 'Count_Person_Male_65OrMoreYears'
+      WHEN variable_measured LIKE '%Female_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Female%' THEN 'Count_Person_65OrMoreYears_Female'
+      WHEN variable_measured LIKE '%Male_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Male%' THEN 'Count_Person_65OrMoreYears_Male'
       WHEN variable_measured LIKE '%65OrMoreYears%' THEN 'Count_Person_65OrMoreYears'
       WHEN variable_measured LIKE '%18To64Years%' THEN 'Count_Person_18To64Years'
       WHEN variable_measured LIKE '%18OrMoreYears%' THEN 'Count_Person_18OrMoreYears'
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index eaec37e2f0..c013d94153 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -62,5 +62,15 @@ def test_run_process_query_error(self):
             with self.assertRaises(Exception):
                 process.run_process(mock_client, output_file)
 
+    def test_run_process_dataframe_error(self):
+        mock_client = mock.MagicMock()
+        mock_query_job = mock.MagicMock()
+        mock_query_job.to_dataframe.side_effect = Exception("Failed to fetch dataframe")
+        mock_client.query.return_value = mock_query_job
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            with self.assertRaises(Exception):
+                process.run_process(mock_client, output_file)
+
 if __name__ == '__main__':
     unittest.main()

From f7d2dfc868e6db08865fff523b16953fb79e164e Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 2 Sep 2026 07:34:48 +0000
Subject: [PATCH 08/44] Return True on successful process completion and update
 unit tests

---
 scripts/us_cdc/cdc500_state/process.py      |  6 +++++-
 scripts/us_cdc/cdc500_state/process_test.py | 11 +++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index f4476810f3..1b39961d6e 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -105,7 +105,8 @@
 GROUP BY 1, 2, 3, 4, 5
 """
 
-def run_process(client: bigquery.Client, output_file: str) -> None:
+
+def run_process(client: bigquery.Client, output_file: str) -> bool:
     """Executes the BigQuery query and writes the resulting DataFrame to output_file."""
     logging.info("Running BigQuery aggregation query...")
     try:
@@ -126,6 +127,8 @@ def run_process(client: bigquery.Client, output_file: str) -> None:
         os.makedirs(output_dir, exist_ok=True)
     logging.info("Writing %d rows to %s", len(df), output_file)
     df.to_csv(output_file, index=False)
+    return True
+
 
 def main(argv):
     del argv  # Unused.
@@ -133,5 +136,6 @@ def main(argv):
     output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
     run_process(client, output_file)
 
+
 if __name__ == '__main__':
     app.run(main)
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index c013d94153..93d18f8274 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -21,6 +21,7 @@
 
 from scripts.us_cdc.cdc500_state import process
 
+
 class CDC500StateProcessTest(unittest.TestCase):
 
     def test_query_constants(self):
@@ -43,11 +44,11 @@ def test_run_process_success(self):
             'percent': [29.6479]
         })
         mock_client.query.return_value.to_dataframe.return_value = sample_data
-        
+
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            process.run_process(mock_client, output_file)
-            
+            result = process.run_process(mock_client, output_file)
+            self.assertTrue(result)
             mock_client.query.assert_called_once()
             self.assertTrue(os.path.exists(output_file))
             saved_df = pd.read_csv(output_file)
@@ -65,12 +66,14 @@ def test_run_process_query_error(self):
     def test_run_process_dataframe_error(self):
         mock_client = mock.MagicMock()
         mock_query_job = mock.MagicMock()
-        mock_query_job.to_dataframe.side_effect = Exception("Failed to fetch dataframe")
+        mock_query_job.to_dataframe.side_effect = Exception(
+            "Failed to fetch dataframe")
         mock_client.query.return_value = mock_query_job
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
             with self.assertRaises(Exception):
                 process.run_process(mock_client, output_file)
 
+
 if __name__ == '__main__':
     unittest.main()

From 475be721966bf21071576ab00ea6ecd607cca95e Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 2 Sep 2026 11:20:56 +0000
Subject: [PATCH 09/44] Restrict entity resolution to city-level places and
 restore source_files in manifest

---
 scripts/us_cdc/cdc500_state/README.md       | 2 +-
 scripts/us_cdc/cdc500_state/manifest.json   | 3 +++
 scripts/us_cdc/cdc500_state/process.py      | 2 ++
 scripts/us_cdc/cdc500_state/process_test.py | 1 +
 4 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index e651276c93..45d26db85c 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -25,7 +25,7 @@ The aggregation script queries Google Cloud BigQuery graph tables in dataset `da
    - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
 
 2. **`Observation`**:
-   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%'`) for CDC 500 StatVars.
+   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%' AND LENGTH(entity1) = 13`) for CDC 500 StatVars.
    - **City Cohort Populations**: Fetches city-level population counts (`value AS population`) for the corresponding demographic cohort StatVars.
 
 ### Aggregation Methodology
diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index 4a2ac714b8..3ea02369d0 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -22,6 +22,9 @@
                 "disk": 100
             },
             "cron_schedule": "0 1 * * 1",
+            "source_files": [
+                "CDC500State_Output/CDC500State_Output.csv"
+            ],
             "validation_config_file": "validation_config.json"
         }
     ]
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 1b39961d6e..e9e7f31836 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -64,6 +64,7 @@
   INNER JOIN cdc_sv
     ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
+    AND LENGTH(O.entity1) = 13
     AND O.variable_measured LIKE 'Percent_%'
 ),
 
@@ -85,6 +86,7 @@
   ) AS pop
     ON O.variable_measured = pop.pop_statvar
   WHERE O.entity1 LIKE 'geoId/%'
+    AND LENGTH(O.entity1) = 13
 )
 
 SELECT 
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 93d18f8274..6660acf1e7 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -32,6 +32,7 @@ def test_query_constants(self):
         self.assertIn("dc/base/CensusACS5YearSurvey", query)
         self.assertIn("SAFE_DIVIDE", query)
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
+        self.assertIn("LENGTH(O.entity1) = 13", query)
 
     def test_run_process_success(self):
         mock_client = mock.MagicMock()

From da5e51cf2ae28614bf78bdf117a64ccb258d2b9c Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 8 Sep 2026 13:01:10 +0000
Subject: [PATCH 10/44] Address code review feedback on CDC500 state
 aggregation

- Exclude age-bracketed cancer screening indicators lacking composite Census ACS cohorts (Mammography, Cervical screening, Pap smear, Colorectal screening).
- Add QUALIFY deduplication in svo_count for 1:1 population weighting.
- Raise RuntimeError on empty query results and write CSV atomically.
- Add scoped SQL_VALIDATOR check for cohort vintage freshness in validation_config.json.
- Remove duplicate source_files in manifest.json and ensure trailing newlines.
- Expand unit tests to verify query constants, error handling, and CLI flags.
- Update README.md with rationale, prerequisites, testing, and automation details.
---
 scripts/us_cdc/cdc500_state/README.md         | 43 +++++++++++++++++--
 scripts/us_cdc/cdc500_state/cdc500_state.tmcf |  2 +-
 scripts/us_cdc/cdc500_state/manifest.json     |  5 +--
 scripts/us_cdc/cdc500_state/process.py        | 34 ++++++++++++---
 scripts/us_cdc/cdc500_state/process_test.py   | 39 +++++++++++++++--
 .../cdc500_state/validation_config.json       |  9 ++++
 6 files changed, 113 insertions(+), 19 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 45d26db85c..033a271b91 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -39,6 +39,16 @@ $$\text{State Percent} = \frac{\sum (\text{City Population} \times \text{City Pe
 
 The output measurement method is prefixed with `dcAggregate/` (e.g., `dcAggregate/CrudePrevalence`).
 
+#### Excluded Indicators
+
+The following age-bracketed cancer screening indicators are omitted from state-level aggregation:
+- `Percent_Person_50To74Years_Female_ReceivedMammography`
+- `Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening`
+- `Percent_Person_21To65Years_Female_ReceivedPapSmearTest`
+- `Percent_Person_50To75Years_ReceivedColorectalCancerScreening`
+
+**Rationale**: The Census ACS 5-Year Survey does not publish single composite population StatVars for these non-standard multi-year age brackets (`50To74Years`, `21To65Years`, `50To75Years`). Rather than applying arbitrary proxy weights or risking silent row omission, these indicators are explicitly excluded from state aggregation.
+
 ## About the Import
 
 ### Artifacts
@@ -57,10 +67,37 @@ The output measurement method is prefixed with `dcAggregate/` (e.g., `dcAggregat
 
 ### Import Procedure
 
-#### Data Download and Processing Steps
+#### Prerequisites
+
+Ensure Google Cloud authentication is configured with access to BigQuery dataset `datcom-store.spanner_dc_graph_prod_DEFAULT`:
+
+```bash
+$ gcloud auth application-default login
+```
+
+#### Running the Script
+
+To run the BigQuery aggregation and write the output CSV to the default output directory (`CDC500State_Output/CDC500State_Output.csv`):
+
+```bash
+$ python3 scripts/us_cdc/cdc500_state/process.py
+```
+
+To specify a custom output directory:
+
+```bash
+$ python3 scripts/us_cdc/cdc500_state/process.py --output_dir=/path/to/output
+```
+
+#### Running Unit Tests
 
-To run the BigQuery aggregation and generate the output CSV:
+Run the test suite using Python's `unittest` runner from the repository root:
 
 ```bash
-$ python3 process.py
+$ python3 -m unittest scripts.us_cdc.cdc500_state.process_test
 ```
+
+#### Automation
+
+This import is automated via Data Commons Import Automation and scheduled to run weekly via Cloud Batch every Monday at 01:00 UTC (`cron_schedule: "0 1 * * 1"` in `manifest.json`).
+
diff --git a/scripts/us_cdc/cdc500_state/cdc500_state.tmcf b/scripts/us_cdc/cdc500_state/cdc500_state.tmcf
index b7d6106b85..e5246d284d 100644
--- a/scripts/us_cdc/cdc500_state/cdc500_state.tmcf
+++ b/scripts/us_cdc/cdc500_state/cdc500_state.tmcf
@@ -7,4 +7,4 @@ value: C:CDC->percent
 unit: Percent
 scalingFactor: 100
 measurementMethod: C:CDC->measurement_method
-observationPeriod: "P1Y"
\ No newline at end of file
+observationPeriod: "P1Y"
diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index 3ea02369d0..3a314ec2a5 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -22,10 +22,7 @@
                 "disk": 100
             },
             "cron_schedule": "0 1 * * 1",
-            "source_files": [
-                "CDC500State_Output/CDC500State_Output.csv"
-            ],
             "validation_config_file": "validation_config.json"
         }
     ]
-}
\ No newline at end of file
+}
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index e9e7f31836..c5489eeaa4 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -31,10 +31,8 @@
   SELECT
     variable_measured AS cdc500,
     CASE
-      WHEN variable_measured LIKE '%Female_50To74Years%' OR variable_measured LIKE '%50To74Years_Female%' THEN 'Count_Person_Female_50To74Years'
-      WHEN variable_measured LIKE '%Female_21To65Years%' OR variable_measured LIKE '%21To65Years_Female%' THEN 'Count_Person_Female_21To65Years'
-      WHEN variable_measured LIKE '%Female_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Female%' THEN 'Count_Person_65OrMoreYears_Female'
-      WHEN variable_measured LIKE '%Male_65OrMoreYears%' OR variable_measured LIKE '%65OrMoreYears_Male%' THEN 'Count_Person_65OrMoreYears_Male'
+      WHEN REGEXP_CONTAINS(variable_measured, r'65OrMoreYears.*Female|Female.*65OrMoreYears') THEN 'Count_Person_65OrMoreYears_Female'
+      WHEN REGEXP_CONTAINS(variable_measured, r'65OrMoreYears.*Male|Male.*65OrMoreYears') THEN 'Count_Person_65OrMoreYears_Male'
       WHEN variable_measured LIKE '%65OrMoreYears%' THEN 'Count_Person_65OrMoreYears'
       WHEN variable_measured LIKE '%18To64Years%' THEN 'Count_Person_18To64Years'
       WHEN variable_measured LIKE '%18OrMoreYears%' THEN 'Count_Person_18OrMoreYears'
@@ -43,6 +41,12 @@
   FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries`
   WHERE provenance = 'dc/base/CDC500'
     AND variable_measured LIKE 'Percent_%'
+    AND variable_measured NOT IN (
+      'Percent_Person_50To74Years_Female_ReceivedMammography',
+      'Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening',
+      'Percent_Person_21To65Years_Female_ReceivedPapSmearTest',
+      'Percent_Person_50To75Years_ReceivedColorectalCancerScreening'
+    )
   GROUP BY cdc500, pop_statvar
 ),
 
@@ -66,6 +70,12 @@
   WHERE O.entity1 LIKE 'geoId/%'
     AND LENGTH(O.entity1) = 13
     AND O.variable_measured LIKE 'Percent_%'
+    AND O.variable_measured NOT IN (
+      'Percent_Person_50To74Years_Female_ReceivedMammography',
+      'Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening',
+      'Percent_Person_21To65Years_Female_ReceivedPapSmearTest',
+      'Percent_Person_50To75Years_ReceivedColorectalCancerScreening'
+    )
 ),
 
 svo_count AS (
@@ -87,6 +97,10 @@
     ON O.variable_measured = pop.pop_statvar
   WHERE O.entity1 LIKE 'geoId/%'
     AND LENGTH(O.entity1) = 13
+  QUALIFY ROW_NUMBER() OVER (
+    PARTITION BY O.variable_measured, O.entity1, O.date
+    ORDER BY O.facet_id DESC
+  ) = 1
 )
 
 SELECT 
@@ -114,21 +128,26 @@ def run_process(client: bigquery.Client, output_file: str) -> bool:
     try:
         query_job = client.query(QUERY)
     except Exception as e:
-        logging.error("Failed to submit BigQuery query: %s", e)
+        logging.error("Failed to submit BigQuery query: %s", e, exc_info=True)
         raise
 
     logging.info("Fetching query results into dataframe...")
     try:
         df = query_job.to_dataframe()
     except Exception as e:
-        logging.error("Failed to fetch query results into dataframe: %s", e)
+        logging.error("Failed to fetch query results into dataframe: %s", e, exc_info=True)
         raise
 
+    if df.empty:
+        raise RuntimeError("BigQuery query returned 0 rows.")
+
     output_dir = os.path.dirname(output_file)
     if output_dir:
         os.makedirs(output_dir, exist_ok=True)
     logging.info("Writing %d rows to %s", len(df), output_file)
-    df.to_csv(output_file, index=False)
+    temp_file = output_file + ".tmp"
+    df.to_csv(temp_file, index=False)
+    os.replace(temp_file, output_file)
     return True
 
 
@@ -141,3 +160,4 @@ def main(argv):
 
 if __name__ == '__main__':
     app.run(main)
+
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 6660acf1e7..1ec0cde445 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -17,10 +17,13 @@
 import tempfile
 import unittest
 from unittest import mock
+from absl import flags
 import pandas as pd
 
 from scripts.us_cdc.cdc500_state import process
 
+FLAGS = flags.FLAGS
+
 
 class CDC500StateProcessTest(unittest.TestCase):
 
@@ -33,6 +36,12 @@ def test_query_constants(self):
         self.assertIn("SAFE_DIVIDE", query)
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
         self.assertIn("LENGTH(O.entity1) = 13", query)
+        self.assertIn("REGEXP_CONTAINS", query)
+        self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
+        self.assertIn("Percent_Person_50To74Years_Female_ReceivedMammography", query)
+        self.assertIn("Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening", query)
+        self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest", query)
+        self.assertIn("Percent_Person_50To75Years_ReceivedColorectalCancerScreening", query)
 
     def test_run_process_success(self):
         mock_client = mock.MagicMock()
@@ -52,29 +61,51 @@ def test_run_process_success(self):
             self.assertTrue(result)
             mock_client.query.assert_called_once()
             self.assertTrue(os.path.exists(output_file))
+            self.assertFalse(os.path.exists(output_file + '.tmp'))
             saved_df = pd.read_csv(output_file)
             self.assertEqual(len(saved_df), 1)
             self.assertEqual(saved_df['observation_about'].iloc[0], 'geoId/06')
 
+    def test_run_process_empty_dataframe_raises_runtime_error(self):
+        mock_client = mock.MagicMock()
+        mock_client.query.return_value.to_dataframe.return_value = pd.DataFrame()
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            with self.assertRaises(RuntimeError):
+                process.run_process(mock_client, output_file)
+
     def test_run_process_query_error(self):
         mock_client = mock.MagicMock()
-        mock_client.query.side_effect = Exception("BigQuery Access Denied")
+        mock_client.query.side_effect = RuntimeError("BigQuery Access Denied")
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaises(Exception):
+            with self.assertRaises(RuntimeError):
                 process.run_process(mock_client, output_file)
 
     def test_run_process_dataframe_error(self):
         mock_client = mock.MagicMock()
         mock_query_job = mock.MagicMock()
-        mock_query_job.to_dataframe.side_effect = Exception(
+        mock_query_job.to_dataframe.side_effect = RuntimeError(
             "Failed to fetch dataframe")
         mock_client.query.return_value = mock_query_job
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaises(Exception):
+            with self.assertRaises(RuntimeError):
                 process.run_process(mock_client, output_file)
 
+    @mock.patch('scripts.us_cdc.cdc500_state.process.run_process')
+    @mock.patch('google.cloud.bigquery.Client')
+    def test_main(self, mock_bq_client_cls, mock_run_process):
+        mock_client_instance = mock.MagicMock()
+        mock_bq_client_cls.return_value = mock_client_instance
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            FLAGS(['test_process', f'--output_dir={tmp_dir}'])
+            process.main([])
+            expected_output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            mock_run_process.assert_called_once_with(mock_client_instance,
+                                                     expected_output_file)
+
 
 if __name__ == '__main__':
     unittest.main()
+
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 5ac0fba24f..3d7b0cc4a2 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -25,6 +25,15 @@
                 "minimum": 50,
                 "maximum": 52
             }
+        },
+        {
+            "rule_id": "check_statvar_max_dates",
+            "description": "Verifies that MaxDate meets expected vintage freshness per StatVar cohort.",
+            "validator": "SQL_VALIDATOR",
+            "params": {
+                "query": "SELECT StatVar, CAST(MaxDate AS INTEGER) AS max_year FROM stats",
+                "condition": "CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN max_year >= 2016 WHEN StatVar LIKE '%CorePreventiveServices%' THEN max_year >= 2020 WHEN StatVar IN ('Percent_Person_WithHighBloodPressure', 'Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication', 'Percent_Person_WithHighCholesterol', 'Percent_Person_ReceivedCholesterolScreening', 'Percent_Person_WithChronicKidneyDisease') THEN max_year >= 2021 ELSE max_year >= 2022 END"
+            }
         }
     ]
 }

From b69963eaf34874e4671e7e6c29e6055b36f4bd19 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 8 Sep 2026 17:37:07 +0000
Subject: [PATCH 11/44] Address code review findings for CDC500 state
 aggregation

- Add check_deleted_records_percent override (threshold 100) in validation_config.json for TMCF unit/scalingFactor migration.
- Remove redundant try/except re-raise logging in run_process to eliminate double tracebacks.
- Add unit tests for demographic cohort regex mappings and population-weighted aggregation.
- Fix demographic cohort examples in README.md to reflect excluded cancer screening indicators.
---
 scripts/us_cdc/cdc500_state/README.md         |  2 +-
 scripts/us_cdc/cdc500_state/process.py        | 12 +---
 scripts/us_cdc/cdc500_state/process_test.py   | 61 +++++++++++++++++++
 .../cdc500_state/validation_config.json       |  8 +++
 4 files changed, 72 insertions(+), 11 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 033a271b91..e2b6da4d34 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -21,7 +21,7 @@ The state-level dataset calculates aggregated health indicator prevalence estima
 The aggregation script queries Google Cloud BigQuery graph tables in dataset `datcom-store.spanner_dc_graph_prod_DEFAULT`:
 
 1. **`TimeSeries`**:
-   - **CDC 500 Series**: Identifies CDC 500 Statistical Variables (`provenance = 'dc/base/CDC500'` and `variable_measured LIKE 'Percent_%'`) and extracts their measurement methods (`measurement_method`). It maps each percentage health metric to its appropriate denominator demographic cohort StatVar (e.g., `Count_Person_18OrMoreYears`, `Count_Person_Female_50To74Years`, `Count_Person_Female_21To65Years`, `Count_Person_65OrMoreYears`, etc.).
+   - **CDC 500 Series**: Identifies CDC 500 Statistical Variables (`provenance = 'dc/base/CDC500'` and `variable_measured LIKE 'Percent_%'`) and extracts their measurement methods (`measurement_method`). It maps each percentage health metric to its appropriate denominator demographic cohort StatVar (e.g., `Count_Person_18OrMoreYears`, `Count_Person_18To64Years`, `Count_Person_65OrMoreYears`, `Count_Person`, etc.).
    - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
 
 2. **`Observation`**:
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index c5489eeaa4..0550882dd2 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -125,18 +125,10 @@
 def run_process(client: bigquery.Client, output_file: str) -> bool:
     """Executes the BigQuery query and writes the resulting DataFrame to output_file."""
     logging.info("Running BigQuery aggregation query...")
-    try:
-        query_job = client.query(QUERY)
-    except Exception as e:
-        logging.error("Failed to submit BigQuery query: %s", e, exc_info=True)
-        raise
+    query_job = client.query(QUERY)
 
     logging.info("Fetching query results into dataframe...")
-    try:
-        df = query_job.to_dataframe()
-    except Exception as e:
-        logging.error("Failed to fetch query results into dataframe: %s", e, exc_info=True)
-        raise
+    df = query_job.to_dataframe()
 
     if df.empty:
         raise RuntimeError("BigQuery query returned 0 rows.")
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 1ec0cde445..6f68339def 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -43,6 +43,67 @@ def test_query_constants(self):
         self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest", query)
         self.assertIn("Percent_Person_50To75Years_ReceivedColorectalCancerScreening", query)
 
+    def test_demographic_cohort_regex_mapping(self):
+        """Verifies that representative StatVars match the intended demographic regex rules."""
+        import re
+        female_pattern = r'65OrMoreYears.*Female|Female.*65OrMoreYears'
+        male_pattern = r'65OrMoreYears.*Male|Male.*65OrMoreYears'
+
+        self.assertIn(female_pattern, process.QUERY)
+        self.assertIn(male_pattern, process.QUERY)
+
+        # Helper mapping that mirrors the SQL CASE WHEN logic
+        def map_statvar(sv: str) -> str:
+            if re.search(female_pattern, sv):
+                return 'Count_Person_65OrMoreYears_Female'
+            elif re.search(male_pattern, sv):
+                return 'Count_Person_65OrMoreYears_Male'
+            elif '65OrMoreYears' in sv:
+                return 'Count_Person_65OrMoreYears'
+            elif '18To64Years' in sv:
+                return 'Count_Person_18To64Years'
+            elif '18OrMoreYears' in sv:
+                return 'Count_Person_18OrMoreYears'
+            else:
+                return 'Count_Person'
+
+        test_cases = [
+            ('Percent_Person_65OrMoreYears_Female_CorePreventiveServices', 'Count_Person_65OrMoreYears_Female'),
+            ('Percent_Person_Female_65OrMoreYears_CorePreventiveServices', 'Count_Person_65OrMoreYears_Female'),
+            ('Percent_Person_65OrMoreYears_Male_CorePreventiveServices', 'Count_Person_65OrMoreYears_Male'),
+            ('Percent_Person_Male_65OrMoreYears_CorePreventiveServices', 'Count_Person_65OrMoreYears_Male'),
+            ('Percent_Person_65OrMoreYears_CorePreventiveServices', 'Count_Person_65OrMoreYears'),
+            ('Percent_Person_18To64Years_HealthInsurance', 'Count_Person_18To64Years'),
+            ('Percent_Person_18OrMoreYears_WithAnyDisability', 'Count_Person_18OrMoreYears'),
+            ('Percent_Person_18OrMoreYears_WithHighBloodPressure', 'Count_Person_18OrMoreYears'),
+            ('Percent_Person_WithArthritis', 'Count_Person'),
+            ('Percent_Person_WithHighCholesterol', 'Count_Person'),
+        ]
+
+        for sv, expected in test_cases:
+            with self.subTest(statvar=sv):
+                self.assertEqual(map_statvar(sv), expected)
+
+    def test_population_weighted_average_calculation(self):
+        """Verifies the population-weighted average calculation and city-to-state FIPS aggregation."""
+        # Simulated city-level records for California (geoId/06)
+        city_records = pd.DataFrame({
+            'city_geoid': ['geoId/0644000', 'geoId/0666000', 'geoId/0667000'],
+            'city_percent': [20.0, 30.0, 40.0],
+            'city_pop': [10000, 20000, 70000]
+        })
+        city_records['state_geoid'] = city_records['city_geoid'].str.slice(0, 8)
+        self.assertTrue((city_records['state_geoid'] == 'geoId/06').all())
+
+        # Formula: SUM(pop * percent) / SUM(pop)
+        total_weighted = (city_records['city_pop'] * city_records['city_percent']).sum()
+        total_pop = city_records['city_pop'].sum()
+        weighted_avg = total_weighted / total_pop
+
+        # Expected: (10000*20 + 20000*30 + 70000*40) / 100000 = (200000 + 600000 + 2800000) / 100000 = 36.0
+        self.assertEqual(total_pop, 100000)
+        self.assertAlmostEqual(weighted_avg, 36.0, places=4)
+
     def test_run_process_success(self):
         mock_client = mock.MagicMock()
         sample_data = pd.DataFrame({
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 3d7b0cc4a2..ad5285bf8a 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -1,6 +1,14 @@
 {
     "schema_version": "1.0",
     "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Override default threshold to 100% due to one-time series identity update (adding unit: Percent and scalingFactor: 100).",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 100
+            }
+        },
         {
             "rule_id": "check_max_value_percentage",
             "description": "Checks that all percentage StatVars do not exceed 100%.",

From 81294e4b4f0324a12098eba4bb20ad5f23ea9819 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 9 Sep 2026 07:33:05 +0000
Subject: [PATCH 12/44] Remove check_statvar_max_dates and
 check_deleted_records_percent from validation_config.json

---
 .../us_cdc/cdc500_state/validation_config.json  | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index ad5285bf8a..5ac0fba24f 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -1,14 +1,6 @@
 {
     "schema_version": "1.0",
     "rules": [
-        {
-            "rule_id": "check_deleted_records_percent",
-            "description": "Override default threshold to 100% due to one-time series identity update (adding unit: Percent and scalingFactor: 100).",
-            "validator": "DELETED_RECORDS_PERCENT",
-            "params": {
-                "threshold": 100
-            }
-        },
         {
             "rule_id": "check_max_value_percentage",
             "description": "Checks that all percentage StatVars do not exceed 100%.",
@@ -33,15 +25,6 @@
                 "minimum": 50,
                 "maximum": 52
             }
-        },
-        {
-            "rule_id": "check_statvar_max_dates",
-            "description": "Verifies that MaxDate meets expected vintage freshness per StatVar cohort.",
-            "validator": "SQL_VALIDATOR",
-            "params": {
-                "query": "SELECT StatVar, CAST(MaxDate AS INTEGER) AS max_year FROM stats",
-                "condition": "CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN max_year >= 2016 WHEN StatVar LIKE '%CorePreventiveServices%' THEN max_year >= 2020 WHEN StatVar IN ('Percent_Person_WithHighBloodPressure', 'Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication', 'Percent_Person_WithHighCholesterol', 'Percent_Person_ReceivedCholesterolScreening', 'Percent_Person_WithChronicKidneyDisease') THEN max_year >= 2021 ELSE max_year >= 2022 END"
-            }
         }
     ]
 }

From c4067f960ebd0926f7fc86a4b7385c770b4f680d Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 9 Sep 2026 08:50:34 +0000
Subject: [PATCH 13/44] Address review findings: add deterministic
 last_update_timestamp deduplication in process.py and use TRY_CAST in
 validation_config.json

---
 scripts/us_cdc/cdc500_state/process.py             | 6 +++++-
 scripts/us_cdc/cdc500_state/process_test.py        | 1 +
 scripts/us_cdc/cdc500_state/validation_config.json | 9 +++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 0550882dd2..1c1f67b567 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -76,6 +76,10 @@
       'Percent_Person_21To65Years_Female_ReceivedPapSmearTest',
       'Percent_Person_50To75Years_ReceivedColorectalCancerScreening'
     )
+  QUALIFY ROW_NUMBER() OVER (
+    PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
+    ORDER BY O.last_update_timestamp DESC
+  ) = 1
 ),
 
 svo_count AS (
@@ -99,7 +103,7 @@
     AND LENGTH(O.entity1) = 13
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date
-    ORDER BY O.facet_id DESC
+    ORDER BY O.last_update_timestamp DESC
   ) = 1
 )
 
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 6f68339def..e5eecaa8cf 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -38,6 +38,7 @@ def test_query_constants(self):
         self.assertIn("LENGTH(O.entity1) = 13", query)
         self.assertIn("REGEXP_CONTAINS", query)
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
+        self.assertIn("O.last_update_timestamp DESC", query)
         self.assertIn("Percent_Person_50To74Years_Female_ReceivedMammography", query)
         self.assertIn("Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening", query)
         self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest", query)
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 5ac0fba24f..1a18f40b33 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -25,6 +25,15 @@
                 "minimum": 50,
                 "maximum": 52
             }
+        },
+        {
+            "rule_id": "check_statvar_max_dates",
+            "description": "Verifies that MaxDate meets expected vintage freshness per StatVar cohort.",
+            "validator": "SQL_VALIDATOR",
+            "params": {
+                "query": "SELECT StatVar, TRY_CAST(MaxDate AS INTEGER) AS max_year FROM stats",
+                "condition": "CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN max_year >= 2016 WHEN StatVar LIKE '%CorePreventiveServices%' THEN max_year >= 2020 WHEN StatVar IN ('Percent_Person_WithHighBloodPressure', 'Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication', 'Percent_Person_WithHighCholesterol', 'Percent_Person_ReceivedCholesterolScreening', 'Percent_Person_WithChronicKidneyDisease') THEN max_year >= 2021 ELSE max_year >= 2022 END"
+            }
         }
     ]
 }

From e9e14fefdcfda8456a6302dc63fac5b8a3f59f8f Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 9 Sep 2026 12:59:32 +0000
Subject: [PATCH 14/44] Restore source_files in CDC500 state manifest.json

---
 scripts/us_cdc/cdc500_state/manifest.json | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index 3a314ec2a5..ff3779d643 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -21,6 +21,9 @@
                 "memory": 64,
                 "disk": 100
             },
+            "source_files": [
+                "CDC500State_Output/CDC500State_Output.csv"
+            ],
             "cron_schedule": "0 1 * * 1",
             "validation_config_file": "validation_config.json"
         }

From 0aa9e78ec5e08fb9ef557ed070f9367a9497c81b Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 10 Sep 2026 11:58:12 +0000
Subject: [PATCH 15/44] Address code review feedback on CDC500 state
 aggregation

- Add validation_config.json to manifest.json source_files
- Add secondary facet_id tie-breaker for deterministic deduplication in process.py
- Use SAFE_CAST in process.py BigQuery aggregation formula
- Add atomic write verification with size check and cleanup in process.py
- Log BigQuery Job ID in process.py
- Move import re to top-level and fix line lengths in process_test.py
- Remove check_statvar_max_dates rule from validation_config.json
- Clarify default output path and demographic cohort weighting in README.md
---
 scripts/us_cdc/cdc500_state/README.md         |  4 ++-
 scripts/us_cdc/cdc500_state/manifest.json     |  3 +-
 scripts/us_cdc/cdc500_state/process.py        | 27 +++++++++++-----
 scripts/us_cdc/cdc500_state/process_test.py   | 31 ++++++++++++-------
 .../cdc500_state/validation_config.json       |  9 ------
 5 files changed, 44 insertions(+), 30 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index e2b6da4d34..02d756c47a 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -39,6 +39,8 @@ $$\text{State Percent} = \frac{\sum (\text{City Population} \times \text{City Pe
 
 The output measurement method is prefixed with `dcAggregate/` (e.g., `dcAggregate/CrudePrevalence`).
 
+*Note on Demographic Cohorts:* Several adult health indicators (such as `Percent_Person_Smoking`, `Percent_Person_Obesity`, `Percent_Person_WithDiabetes`) lack explicit age tokens in their DCID and intentionally fall back to `Count_Person` weighting to preserve bitwise parity with existing Data Commons baseline observations.
+
 #### Excluded Indicators
 
 The following age-bracketed cancer screening indicators are omitted from state-level aggregation:
@@ -77,7 +79,7 @@ $ gcloud auth application-default login
 
 #### Running the Script
 
-To run the BigQuery aggregation and write the output CSV to the default output directory (`CDC500State_Output/CDC500State_Output.csv`):
+To run the BigQuery aggregation and write the output CSV to the default output directory (`scripts/us_cdc/cdc500_state/CDC500State_Output/CDC500State_Output.csv` when run from repository root):
 
 ```bash
 $ python3 scripts/us_cdc/cdc500_state/process.py
diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index ff3779d643..a9176803e8 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -22,7 +22,8 @@
                 "disk": 100
             },
             "source_files": [
-                "CDC500State_Output/CDC500State_Output.csv"
+                "CDC500State_Output/CDC500State_Output.csv",
+                "validation_config.json"
             ],
             "cron_schedule": "0 1 * * 1",
             "validation_config_file": "validation_config.json"
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 1c1f67b567..fd05619117 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -31,8 +31,12 @@
   SELECT
     variable_measured AS cdc500,
     CASE
-      WHEN REGEXP_CONTAINS(variable_measured, r'65OrMoreYears.*Female|Female.*65OrMoreYears') THEN 'Count_Person_65OrMoreYears_Female'
-      WHEN REGEXP_CONTAINS(variable_measured, r'65OrMoreYears.*Male|Male.*65OrMoreYears') THEN 'Count_Person_65OrMoreYears_Male'
+      WHEN REGEXP_CONTAINS(
+        variable_measured, r'65OrMoreYears.*Female|Female.*65OrMoreYears'
+      ) THEN 'Count_Person_65OrMoreYears_Female'
+      WHEN REGEXP_CONTAINS(
+        variable_measured, r'65OrMoreYears.*Male|Male.*65OrMoreYears'
+      ) THEN 'Count_Person_65OrMoreYears_Male'
       WHEN variable_measured LIKE '%65OrMoreYears%' THEN 'Count_Person_65OrMoreYears'
       WHEN variable_measured LIKE '%18To64Years%' THEN 'Count_Person_18To64Years'
       WHEN variable_measured LIKE '%18OrMoreYears%' THEN 'Count_Person_18OrMoreYears'
@@ -78,7 +82,7 @@
     )
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
-    ORDER BY O.last_update_timestamp DESC
+    ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
   ) = 1
 ),
 
@@ -103,7 +107,7 @@
     AND LENGTH(O.entity1) = 13
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date
-    ORDER BY O.last_update_timestamp DESC
+    ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
   ) = 1
 )
 
@@ -114,8 +118,8 @@
   CONCAT('dcAggregate/', p.measurement_method) AS measurement_method,
   p.pop_statvar AS population_statvar,
   SAFE_DIVIDE(
-    SUM(CAST(c.population AS FLOAT64) * CAST(p.percent AS FLOAT64)),
-    SUM(CAST(c.population AS FLOAT64))
+    SUM(SAFE_CAST(c.population AS FLOAT64) * SAFE_CAST(p.percent AS FLOAT64)),
+    SUM(SAFE_CAST(c.population AS FLOAT64))
   ) AS percent
 FROM svo_percent AS p
 INNER JOIN svo_count AS c
@@ -130,6 +134,7 @@ def run_process(client: bigquery.Client, output_file: str) -> bool:
     """Executes the BigQuery query and writes the resulting DataFrame to output_file."""
     logging.info("Running BigQuery aggregation query...")
     query_job = client.query(QUERY)
+    logging.info("BigQuery job started with ID: %s", query_job.job_id)
 
     logging.info("Fetching query results into dataframe...")
     df = query_job.to_dataframe()
@@ -142,8 +147,14 @@ def run_process(client: bigquery.Client, output_file: str) -> bool:
         os.makedirs(output_dir, exist_ok=True)
     logging.info("Writing %d rows to %s", len(df), output_file)
     temp_file = output_file + ".tmp"
-    df.to_csv(temp_file, index=False)
-    os.replace(temp_file, output_file)
+    try:
+        df.to_csv(temp_file, index=False)
+        if not os.path.exists(temp_file) or os.path.getsize(temp_file) == 0:
+            raise RuntimeError(f"Output file {temp_file} was created empty or missing.")
+        os.replace(temp_file, output_file)
+    finally:
+        if os.path.exists(temp_file):
+            os.remove(temp_file)
     return True
 
 
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index e5eecaa8cf..347cd4940e 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -14,6 +14,7 @@
 """Unit tests for CDC 500 State aggregation script."""
 
 import os
+import re
 import tempfile
 import unittest
 from unittest import mock
@@ -34,19 +35,20 @@ def test_query_constants(self):
         self.assertIn("dc/base/CDC500", query)
         self.assertIn("dc/base/CensusACS5YearSurvey", query)
         self.assertIn("SAFE_DIVIDE", query)
+        self.assertIn("SAFE_CAST", query)
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
         self.assertIn("LENGTH(O.entity1) = 13", query)
         self.assertIn("REGEXP_CONTAINS", query)
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
-        self.assertIn("O.last_update_timestamp DESC", query)
+        self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)
         self.assertIn("Percent_Person_50To74Years_Female_ReceivedMammography", query)
-        self.assertIn("Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening", query)
+        self.assertIn(
+            "Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening", query)
         self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest", query)
         self.assertIn("Percent_Person_50To75Years_ReceivedColorectalCancerScreening", query)
 
     def test_demographic_cohort_regex_mapping(self):
         """Verifies that representative StatVars match the intended demographic regex rules."""
-        import re
         female_pattern = r'65OrMoreYears.*Female|Female.*65OrMoreYears'
         male_pattern = r'65OrMoreYears.*Male|Male.*65OrMoreYears'
 
@@ -69,14 +71,20 @@ def map_statvar(sv: str) -> str:
                 return 'Count_Person'
 
         test_cases = [
-            ('Percent_Person_65OrMoreYears_Female_CorePreventiveServices', 'Count_Person_65OrMoreYears_Female'),
-            ('Percent_Person_Female_65OrMoreYears_CorePreventiveServices', 'Count_Person_65OrMoreYears_Female'),
-            ('Percent_Person_65OrMoreYears_Male_CorePreventiveServices', 'Count_Person_65OrMoreYears_Male'),
-            ('Percent_Person_Male_65OrMoreYears_CorePreventiveServices', 'Count_Person_65OrMoreYears_Male'),
-            ('Percent_Person_65OrMoreYears_CorePreventiveServices', 'Count_Person_65OrMoreYears'),
+            ('Percent_Person_65OrMoreYears_Female_CorePreventiveServices',
+             'Count_Person_65OrMoreYears_Female'),
+            ('Percent_Person_Female_65OrMoreYears_CorePreventiveServices',
+             'Count_Person_65OrMoreYears_Female'),
+            ('Percent_Person_65OrMoreYears_Male_CorePreventiveServices',
+             'Count_Person_65OrMoreYears_Male'),
+            ('Percent_Person_Male_65OrMoreYears_CorePreventiveServices',
+             'Count_Person_65OrMoreYears_Male'),
+            ('Percent_Person_65OrMoreYears_CorePreventiveServices',
+             'Count_Person_65OrMoreYears'),
             ('Percent_Person_18To64Years_HealthInsurance', 'Count_Person_18To64Years'),
             ('Percent_Person_18OrMoreYears_WithAnyDisability', 'Count_Person_18OrMoreYears'),
-            ('Percent_Person_18OrMoreYears_WithHighBloodPressure', 'Count_Person_18OrMoreYears'),
+            ('Percent_Person_18OrMoreYears_WithHighBloodPressure',
+             'Count_Person_18OrMoreYears'),
             ('Percent_Person_WithArthritis', 'Count_Person'),
             ('Percent_Person_WithHighCholesterol', 'Count_Person'),
         ]
@@ -86,7 +94,8 @@ def map_statvar(sv: str) -> str:
                 self.assertEqual(map_statvar(sv), expected)
 
     def test_population_weighted_average_calculation(self):
-        """Verifies the population-weighted average calculation and city-to-state FIPS aggregation."""
+        """Verifies the population-weighted average calculation and city-to-state
+        FIPS aggregation."""
         # Simulated city-level records for California (geoId/06)
         city_records = pd.DataFrame({
             'city_geoid': ['geoId/0644000', 'geoId/0666000', 'geoId/0667000'],
@@ -101,7 +110,7 @@ def test_population_weighted_average_calculation(self):
         total_pop = city_records['city_pop'].sum()
         weighted_avg = total_weighted / total_pop
 
-        # Expected: (10000*20 + 20000*30 + 70000*40) / 100000 = (200000 + 600000 + 2800000) / 100000 = 36.0
+        # Expected: (10000*20 + 20000*30 + 70000*40) / 100000 = 36.0
         self.assertEqual(total_pop, 100000)
         self.assertAlmostEqual(weighted_avg, 36.0, places=4)
 
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 1a18f40b33..5ac0fba24f 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -25,15 +25,6 @@
                 "minimum": 50,
                 "maximum": 52
             }
-        },
-        {
-            "rule_id": "check_statvar_max_dates",
-            "description": "Verifies that MaxDate meets expected vintage freshness per StatVar cohort.",
-            "validator": "SQL_VALIDATOR",
-            "params": {
-                "query": "SELECT StatVar, TRY_CAST(MaxDate AS INTEGER) AS max_year FROM stats",
-                "condition": "CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN max_year >= 2016 WHEN StatVar LIKE '%CorePreventiveServices%' THEN max_year >= 2020 WHEN StatVar IN ('Percent_Person_WithHighBloodPressure', 'Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication', 'Percent_Person_WithHighCholesterol', 'Percent_Person_ReceivedCholesterolScreening', 'Percent_Person_WithChronicKidneyDisease') THEN max_year >= 2021 ELSE max_year >= 2022 END"
-            }
         }
     ]
 }

From 5bdc3752cf00582b6b1c73606f8b7543d4142f29 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 10 Sep 2026 12:12:48 +0000
Subject: [PATCH 16/44] Harden process.py, test suite, and validation config

- Add optional --project CLI flag to process.py for BigQuery client
- Guard measurement_method CONCAT with COALESCE in process.py SQL
- Use absl.testing.flagsaver in process_test.py to avoid global flag pollution
- Verify query parameter in test_run_process_success
- Add comprehensive docstrings and clean import grouping in process_test.py
- Add explicit variable scope to percentage bounds in validation_config.json
---
 scripts/us_cdc/cdc500_state/process.py        | 10 +++++--
 scripts/us_cdc/cdc500_state/process_test.py   | 28 +++++++++++++++----
 .../cdc500_state/validation_config.json       | 10 +++++++
 3 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index fd05619117..87aa062bf3 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -14,6 +14,7 @@
 """Processes CDC 500 cities data into aggregated state-level health indicators."""
 
 import os
+
 from absl import app
 from absl import flags
 from absl import logging
@@ -25,6 +26,9 @@
 
 flags.DEFINE_string('output_dir', _DEFAULT_OUTPUT_DIR,
                     'Directory to write output CSV.')
+flags.DEFINE_string(
+    'project', None,
+    'GCP project ID for BigQuery. Defaults to ambient environment if omitted.')
 
 QUERY = """
 WITH cdc_sv AS (
@@ -115,7 +119,9 @@
   p.statvar, 
   SUBSTR(p.observation_about, 1, 8) AS observation_about,
   p.observation_date, 
-  CONCAT('dcAggregate/', p.measurement_method) AS measurement_method,
+  COALESCE(
+    CONCAT('dcAggregate/', p.measurement_method), 'dcAggregate'
+  ) AS measurement_method,
   p.pop_statvar AS population_statvar,
   SAFE_DIVIDE(
     SUM(SAFE_CAST(c.population AS FLOAT64) * SAFE_CAST(p.percent AS FLOAT64)),
@@ -160,7 +166,7 @@ def run_process(client: bigquery.Client, output_file: str) -> bool:
 
 def main(argv):
     del argv  # Unused.
-    client = bigquery.Client()
+    client = bigquery.Client(project=_FLAGS.project)
     output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
     run_process(client, output_file)
 
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 347cd4940e..31ac5bc918 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -18,7 +18,9 @@
 import tempfile
 import unittest
 from unittest import mock
+
 from absl import flags
+from absl.testing import flagsaver
 import pandas as pd
 
 from scripts.us_cdc.cdc500_state import process
@@ -27,8 +29,16 @@
 
 
 class CDC500StateProcessTest(unittest.TestCase):
+    """Unit tests for CDC 500 State aggregation processing."""
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        if not FLAGS.is_parsed():
+            FLAGS(['test'])
 
     def test_query_constants(self):
+        """Verifies key SQL clauses and excluded variables in process.QUERY."""
         query = process.QUERY
         self.assertIn("spanner_dc_graph_prod_DEFAULT.TimeSeries", query)
         self.assertIn("spanner_dc_graph_prod_DEFAULT.Observation", query)
@@ -115,6 +125,7 @@ def test_population_weighted_average_calculation(self):
         self.assertAlmostEqual(weighted_avg, 36.0, places=4)
 
     def test_run_process_success(self):
+        """Tests successful query execution and atomic output writing."""
         mock_client = mock.MagicMock()
         sample_data = pd.DataFrame({
             'statvar': ['Percent_Person_18OrMoreYears_WithAnyDisability'],
@@ -130,7 +141,7 @@ def test_run_process_success(self):
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
             result = process.run_process(mock_client, output_file)
             self.assertTrue(result)
-            mock_client.query.assert_called_once()
+            mock_client.query.assert_called_once_with(process.QUERY)
             self.assertTrue(os.path.exists(output_file))
             self.assertFalse(os.path.exists(output_file + '.tmp'))
             saved_df = pd.read_csv(output_file)
@@ -138,6 +149,7 @@ def test_run_process_success(self):
             self.assertEqual(saved_df['observation_about'].iloc[0], 'geoId/06')
 
     def test_run_process_empty_dataframe_raises_runtime_error(self):
+        """Tests that empty query results raise RuntimeError."""
         mock_client = mock.MagicMock()
         mock_client.query.return_value.to_dataframe.return_value = pd.DataFrame()
         with tempfile.TemporaryDirectory() as tmp_dir:
@@ -146,6 +158,7 @@ def test_run_process_empty_dataframe_raises_runtime_error(self):
                 process.run_process(mock_client, output_file)
 
     def test_run_process_query_error(self):
+        """Tests propagation of BigQuery query execution errors."""
         mock_client = mock.MagicMock()
         mock_client.query.side_effect = RuntimeError("BigQuery Access Denied")
         with tempfile.TemporaryDirectory() as tmp_dir:
@@ -154,6 +167,7 @@ def test_run_process_query_error(self):
                 process.run_process(mock_client, output_file)
 
     def test_run_process_dataframe_error(self):
+        """Tests propagation of query result download errors."""
         mock_client = mock.MagicMock()
         mock_query_job = mock.MagicMock()
         mock_query_job.to_dataframe.side_effect = RuntimeError(
@@ -167,14 +181,16 @@ def test_run_process_dataframe_error(self):
     @mock.patch('scripts.us_cdc.cdc500_state.process.run_process')
     @mock.patch('google.cloud.bigquery.Client')
     def test_main(self, mock_bq_client_cls, mock_run_process):
+        """Tests process.main flag parsing and client instantiation."""
         mock_client_instance = mock.MagicMock()
         mock_bq_client_cls.return_value = mock_client_instance
         with tempfile.TemporaryDirectory() as tmp_dir:
-            FLAGS(['test_process', f'--output_dir={tmp_dir}'])
-            process.main([])
-            expected_output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            mock_run_process.assert_called_once_with(mock_client_instance,
-                                                     expected_output_file)
+            with flagsaver.flagsaver(output_dir=tmp_dir, project='test-project'):
+                process.main([])
+                expected_output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+                mock_bq_client_cls.assert_called_once_with(project='test-project')
+                mock_run_process.assert_called_once_with(mock_client_instance,
+                                                         expected_output_file)
 
 
 if __name__ == '__main__':
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 5ac0fba24f..a7ffeccc10 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -7,6 +7,11 @@
             "validator": "MAX_VALUE_CHECK",
             "params": {
                 "maximum": 100
+            },
+            "scope": {
+                "variables": {
+                    "contains_all": ["Percent"]
+                }
             }
         },
         {
@@ -15,6 +20,11 @@
             "validator": "MIN_VALUE_CHECK",
             "params": {
                 "minimum": 0
+            },
+            "scope": {
+                "variables": {
+                    "contains_all": ["Percent"]
+                }
             }
         },
         {

From 65f4cb056d29b926110fd1bc72cc85b9c951e8ec Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 10 Sep 2026 12:21:10 +0000
Subject: [PATCH 17/44] Polish README artifact links and remove trailing EOF
 newline in test suite

---
 scripts/us_cdc/cdc500_state/README.md       | 14 ++++++++++----
 scripts/us_cdc/cdc500_state/process_test.py |  1 -
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 02d756c47a..85b942e781 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -56,16 +56,16 @@ The following age-bracketed cancer screening indicators are omitted from state-l
 ### Artifacts
 
 #### Scripts
-[`process.py`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/process.py)
+[`process.py`](process.py)
 
 #### Unit Tests
-[`process_test.py`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/process_test.py)
+[`process_test.py`](process_test.py)
 
 #### tMCF Template
-[`cdc500_state.tmcf`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/cdc500_state.tmcf)
+[`cdc500_state.tmcf`](cdc500_state.tmcf)
 
 #### Validation Config
-[`validation_config.json`](https://github.com/datacommonsorg/data/blob/master/scripts/us_cdc/cdc500_state/validation_config.json)
+[`validation_config.json`](validation_config.json)
 
 ### Import Procedure
 
@@ -91,6 +91,12 @@ To specify a custom output directory:
 $ python3 scripts/us_cdc/cdc500_state/process.py --output_dir=/path/to/output
 ```
 
+To specify a GCP project for BigQuery jobs (if not set in ambient environment):
+
+```bash
+$ python3 scripts/us_cdc/cdc500_state/process.py --project=my-gcp-project
+```
+
 #### Running Unit Tests
 
 Run the test suite using Python's `unittest` runner from the repository root:
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 31ac5bc918..7375a2ba85 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -195,4 +195,3 @@ def test_main(self, mock_bq_client_cls, mock_run_process):
 
 if __name__ == '__main__':
     unittest.main()
-

From 7a9c9a72b4135e731cfbc41c4f8c10c38b0f514f Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 10 Sep 2026 12:39:32 +0000
Subject: [PATCH 18/44] Fix line length in validation_config.json to conform to
 PEP 8

---
 scripts/us_cdc/cdc500_state/validation_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index a7ffeccc10..3876fcbde1 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -29,7 +29,7 @@
         },
         {
             "rule_id": "check_num_places_state_count",
-            "description": "Checks that state-level observations cover all 50-52 US state entities.",
+            "description": "Checks that state-level observations cover all 50-52 US states.",
             "validator": "NUM_PLACES_COUNT",
             "params": {
                 "minimum": 50,

From 9e18e01b384b2073f2f648c541d27d966994fe5c Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 10 Sep 2026 12:44:53 +0000
Subject: [PATCH 19/44] Clean up redundant SQL filter and expand unit test
 assertions

---
 scripts/us_cdc/cdc500_state/process.py      |  6 ----
 scripts/us_cdc/cdc500_state/process_test.py | 34 +++++++++++++++++++--
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 87aa062bf3..a9b80d1418 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -78,12 +78,6 @@
   WHERE O.entity1 LIKE 'geoId/%'
     AND LENGTH(O.entity1) = 13
     AND O.variable_measured LIKE 'Percent_%'
-    AND O.variable_measured NOT IN (
-      'Percent_Person_50To74Years_Female_ReceivedMammography',
-      'Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening',
-      'Percent_Person_21To65Years_Female_ReceivedPapSmearTest',
-      'Percent_Person_50To75Years_ReceivedColorectalCancerScreening'
-    )
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
     ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 7375a2ba85..bb22f6b66c 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -52,10 +52,18 @@ def test_query_constants(self):
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
         self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)
         self.assertIn("Percent_Person_50To74Years_Female_ReceivedMammography", query)
-        self.assertIn(
-            "Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening", query)
         self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest", query)
-        self.assertIn("Percent_Person_50To75Years_ReceivedColorectalCancerScreening", query)
+        self.assertIn(
+            "Percent_Person_50To75Years_ReceivedColorectalCancerScreening", query)
+        self.assertIn("'Count_Person_65OrMoreYears_Female'", query)
+        self.assertIn("'Count_Person_65OrMoreYears_Male'", query)
+        self.assertIn("'Count_Person_65OrMoreYears'", query)
+        self.assertIn("'Count_Person_18To64Years'", query)
+        self.assertIn("'Count_Person_18OrMoreYears'", query)
+        self.assertIn("'Count_Person'", query)
+        self.assertIn(
+            "SUM(SAFE_CAST(c.population AS FLOAT64) * "
+            "SAFE_CAST(p.percent AS FLOAT64))", query)
 
     def test_demographic_cohort_regex_mapping(self):
         """Verifies that representative StatVars match the intended demographic regex rules."""
@@ -178,6 +186,26 @@ def test_run_process_dataframe_error(self):
             with self.assertRaises(RuntimeError):
                 process.run_process(mock_client, output_file)
 
+    def test_run_process_empty_output_file_raises_runtime_error(self):
+        """Tests that creating an empty (0-byte) output file raises RuntimeError."""
+        mock_client = mock.MagicMock()
+        mock_df = mock.MagicMock()
+        mock_df.__len__.return_value = 1
+
+        def fake_to_csv(filepath, index=False):
+            del index  # Unused.
+            with open(filepath, 'w'):
+                pass  # Create 0-byte file
+
+        mock_df.to_csv.side_effect = fake_to_csv
+        mock_client.query.return_value.to_dataframe.return_value = mock_df
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            with self.assertRaises(RuntimeError):
+                process.run_process(mock_client, output_file)
+            self.assertFalse(os.path.exists(output_file))
+            self.assertFalse(os.path.exists(output_file + '.tmp'))
+
     @mock.patch('scripts.us_cdc.cdc500_state.process.run_process')
     @mock.patch('google.cloud.bigquery.Client')
     def test_main(self, mock_bq_client_cls, mock_run_process):

From 9c1f0f3bf8f8e55add70213e0730232e251ab370 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 15 Sep 2026 08:39:12 +0000
Subject: [PATCH 20/44] Remove LENGTH(O.entity1) = 13 filter from CDC500 state
 query

---
 scripts/us_cdc/cdc500_state/README.md       | 2 +-
 scripts/us_cdc/cdc500_state/process.py      | 2 --
 scripts/us_cdc/cdc500_state/process_test.py | 2 +-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 85b942e781..2adc3f63a8 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -25,7 +25,7 @@ The aggregation script queries Google Cloud BigQuery graph tables in dataset `da
    - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
 
 2. **`Observation`**:
-   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%' AND LENGTH(entity1) = 13`) for CDC 500 StatVars.
+   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%'`) for CDC 500 StatVars.
    - **City Cohort Populations**: Fetches city-level population counts (`value AS population`) for the corresponding demographic cohort StatVars.
 
 ### Aggregation Methodology
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index a9b80d1418..d325b09c89 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -76,7 +76,6 @@
   INNER JOIN cdc_sv
     ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
-    AND LENGTH(O.entity1) = 13
     AND O.variable_measured LIKE 'Percent_%'
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
@@ -102,7 +101,6 @@
   ) AS pop
     ON O.variable_measured = pop.pop_statvar
   WHERE O.entity1 LIKE 'geoId/%'
-    AND LENGTH(O.entity1) = 13
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date
     ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index bb22f6b66c..818dcde58c 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -47,7 +47,7 @@ def test_query_constants(self):
         self.assertIn("SAFE_DIVIDE", query)
         self.assertIn("SAFE_CAST", query)
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
-        self.assertIn("LENGTH(O.entity1) = 13", query)
+        self.assertNotIn("LENGTH(O.entity1)", query)
         self.assertIn("REGEXP_CONTAINS", query)
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
         self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)

From 628b6c5701a799f3f0a9b6f120b520a7e150fd14 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 15 Sep 2026 09:43:33 +0000
Subject: [PATCH 21/44] Add AND (LENGTH(O.entity1) = 13 OR O.entity1 =
 'geoId/15003') filter to CDC500 state query

---
 scripts/us_cdc/cdc500_state/README.md       | 2 +-
 scripts/us_cdc/cdc500_state/process.py      | 2 ++
 scripts/us_cdc/cdc500_state/process_test.py | 3 ++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 2adc3f63a8..244c46c292 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -25,7 +25,7 @@ The aggregation script queries Google Cloud BigQuery graph tables in dataset `da
    - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
 
 2. **`Observation`**:
-   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%'`) for CDC 500 StatVars.
+   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%' AND (LENGTH(entity1) = 13 OR entity1 = 'geoId/15003')`) for CDC 500 StatVars (including `geoId/15003` for Honolulu, Hawaii).
    - **City Cohort Populations**: Fetches city-level population counts (`value AS population`) for the corresponding demographic cohort StatVars.
 
 ### Aggregation Methodology
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index d325b09c89..943be1ad22 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -76,6 +76,7 @@
   INNER JOIN cdc_sv
     ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
+    AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')
     AND O.variable_measured LIKE 'Percent_%'
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
@@ -101,6 +102,7 @@
   ) AS pop
     ON O.variable_measured = pop.pop_statvar
   WHERE O.entity1 LIKE 'geoId/%'
+    AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date
     ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 818dcde58c..8a0304bc88 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -47,7 +47,8 @@ def test_query_constants(self):
         self.assertIn("SAFE_DIVIDE", query)
         self.assertIn("SAFE_CAST", query)
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
-        self.assertNotIn("LENGTH(O.entity1)", query)
+        self.assertIn(
+            "AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')", query)
         self.assertIn("REGEXP_CONTAINS", query)
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
         self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)

From 356af44438ac922f9069c55842f8c143ac88a98c Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 15 Sep 2026 11:32:53 +0000
Subject: [PATCH 22/44] Fix Hawaii geoId/15003 double-counting, add error
 logging, right-size manifest limits, and strengthen unit tests

---
 scripts/us_cdc/cdc500_state/README.md       |  2 +-
 scripts/us_cdc/cdc500_state/manifest.json   |  6 ++--
 scripts/us_cdc/cdc500_state/process.py      | 33 ++++++++++++++++++---
 scripts/us_cdc/cdc500_state/process_test.py | 16 ++++++----
 4 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 244c46c292..fccbad34f7 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -25,7 +25,7 @@ The aggregation script queries Google Cloud BigQuery graph tables in dataset `da
    - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
 
 2. **`Observation`**:
-   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%' AND (LENGTH(entity1) = 13 OR entity1 = 'geoId/15003')`) for CDC 500 StatVars (including `geoId/15003` for Honolulu, Hawaii).
+   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%' AND (LENGTH(entity1) = 13 OR entity1 = 'geoId/15003')`) for CDC 500 StatVars. To prevent double-counting Oahu/Honolulu County population in years where 13-character Census Designated Places (`geoId/15XXXXX`) are published (2017–2022), `geoId/15003` is dynamically included via a SQL window check only when zero 13-character CDPs exist for Hawaii (`geoId/15`) in that `(StatVar, date, measurement_method)` partition (preserving coverage for 2013–2016 and county-only 2017 indicators).
    - **City Cohort Populations**: Fetches city-level population counts (`value AS population`) for the corresponding demographic cohort StatVars.
 
 ### Aggregation Methodology
diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index a9176803e8..3053c3c89b 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -17,9 +17,9 @@
                 }
             ],
             "resource_limits": {
-                "cpu": 8,
-                "memory": 64,
-                "disk": 100
+                "cpu": 2,
+                "memory": 8,
+                "disk": 20
             },
             "source_files": [
                 "CDC500State_Output/CDC500State_Output.csv",
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 943be1ad22..14f3282c0e 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -58,7 +58,7 @@
   GROUP BY cdc500, pop_statvar
 ),
 
-svo_percent AS (
+svo_percent_dedup AS (
   SELECT
     O.variable_measured AS statvar,
     O.entity1 AS observation_about,
@@ -84,6 +84,28 @@
   ) = 1
 ),
 
+svo_percent AS (
+  SELECT
+    statvar,
+    observation_about,
+    observation_date,
+    percent,
+    measurement_method,
+    pop_statvar
+  FROM svo_percent_dedup
+  QUALIFY LENGTH(observation_about) = 13
+    OR (
+      observation_about = 'geoId/15003'
+      AND COUNTIF(LENGTH(observation_about) = 13) OVER (
+        PARTITION BY
+          statvar,
+          SUBSTR(observation_about, 1, 8),
+          observation_date,
+          measurement_method
+      ) = 0
+    )
+),
+
 svo_count AS (
   SELECT
     O.variable_measured AS population_statvar,
@@ -116,7 +138,6 @@
   COALESCE(
     CONCAT('dcAggregate/', p.measurement_method), 'dcAggregate'
   ) AS measurement_method,
-  p.pop_statvar AS population_statvar,
   SAFE_DIVIDE(
     SUM(SAFE_CAST(c.population AS FLOAT64) * SAFE_CAST(p.percent AS FLOAT64)),
     SUM(SAFE_CAST(c.population AS FLOAT64))
@@ -126,7 +147,7 @@
   ON p.observation_about = c.observation_about
   AND p.observation_date = c.observation_date
   AND p.pop_statvar = c.population_statvar
-GROUP BY 1, 2, 3, 4, 5
+GROUP BY 1, 2, 3, 4
 """
 
 
@@ -140,6 +161,7 @@ def run_process(client: bigquery.Client, output_file: str) -> bool:
     df = query_job.to_dataframe()
 
     if df.empty:
+        logging.error("BigQuery query returned 0 rows.")
         raise RuntimeError("BigQuery query returned 0 rows.")
 
     output_dir = os.path.dirname(output_file)
@@ -150,7 +172,10 @@ def run_process(client: bigquery.Client, output_file: str) -> bool:
     try:
         df.to_csv(temp_file, index=False)
         if not os.path.exists(temp_file) or os.path.getsize(temp_file) == 0:
-            raise RuntimeError(f"Output file {temp_file} was created empty or missing.")
+            logging.error("Output file %s was created empty or missing.",
+                          temp_file)
+            raise RuntimeError(
+                f"Output file {temp_file} was created empty or missing.")
         os.replace(temp_file, output_file)
     finally:
         if os.path.exists(temp_file):
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 8a0304bc88..665bc2725b 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -49,6 +49,7 @@ def test_query_constants(self):
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
         self.assertIn(
             "AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')", query)
+        self.assertIn("COUNTIF(LENGTH(observation_about) = 13) OVER", query)
         self.assertIn("REGEXP_CONTAINS", query)
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
         self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)
@@ -65,6 +66,7 @@ def test_query_constants(self):
         self.assertIn(
             "SUM(SAFE_CAST(c.population AS FLOAT64) * "
             "SAFE_CAST(p.percent AS FLOAT64))", query)
+        self.assertNotIn("p.pop_statvar AS population_statvar", query)
 
     def test_demographic_cohort_regex_mapping(self):
         """Verifies that representative StatVars match the intended demographic regex rules."""
@@ -141,7 +143,6 @@ def test_run_process_success(self):
             'observation_about': ['geoId/06'],
             'observation_date': ['2022'],
             'measurement_method': ['dcAggregate/CrudePrevalence'],
-            'population_statvar': ['Count_Person_18OrMoreYears'],
             'percent': [29.6479]
         })
         mock_client.query.return_value.to_dataframe.return_value = sample_data
@@ -163,7 +164,8 @@ def test_run_process_empty_dataframe_raises_runtime_error(self):
         mock_client.query.return_value.to_dataframe.return_value = pd.DataFrame()
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaises(RuntimeError):
+            with self.assertRaisesRegex(RuntimeError,
+                                        "BigQuery query returned 0 rows"):
                 process.run_process(mock_client, output_file)
 
     def test_run_process_query_error(self):
@@ -172,7 +174,7 @@ def test_run_process_query_error(self):
         mock_client.query.side_effect = RuntimeError("BigQuery Access Denied")
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaises(RuntimeError):
+            with self.assertRaisesRegex(RuntimeError, "BigQuery Access Denied"):
                 process.run_process(mock_client, output_file)
 
     def test_run_process_dataframe_error(self):
@@ -184,13 +186,15 @@ def test_run_process_dataframe_error(self):
         mock_client.query.return_value = mock_query_job
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaises(RuntimeError):
+            with self.assertRaisesRegex(RuntimeError,
+                                        "Failed to fetch dataframe"):
                 process.run_process(mock_client, output_file)
 
     def test_run_process_empty_output_file_raises_runtime_error(self):
         """Tests that creating an empty (0-byte) output file raises RuntimeError."""
         mock_client = mock.MagicMock()
         mock_df = mock.MagicMock()
+        mock_df.empty = False
         mock_df.__len__.return_value = 1
 
         def fake_to_csv(filepath, index=False):
@@ -202,7 +206,9 @@ def fake_to_csv(filepath, index=False):
         mock_client.query.return_value.to_dataframe.return_value = mock_df
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaises(RuntimeError):
+            with self.assertRaisesRegex(
+                    RuntimeError,
+                    "was created empty or missing"):
                 process.run_process(mock_client, output_file)
             self.assertFalse(os.path.exists(output_file))
             self.assertFalse(os.path.exists(output_file + '.tmp'))

From 5ae139bbfcb6e4cc1e063a0cc9cbd6db53f68adb Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 15 Sep 2026 13:46:24 +0000
Subject: [PATCH 23/44] Use explicit WHERE filter for Hawaii geoId/15003 in
 CDC500 state query

---
 scripts/us_cdc/cdc500_state/README.md       |  2 +-
 scripts/us_cdc/cdc500_state/process.py      | 43 +++++++++------------
 scripts/us_cdc/cdc500_state/process_test.py |  5 ++-
 3 files changed, 24 insertions(+), 26 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index fccbad34f7..6180c641f0 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -25,7 +25,7 @@ The aggregation script queries Google Cloud BigQuery graph tables in dataset `da
    - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
 
 2. **`Observation`**:
-   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%' AND (LENGTH(entity1) = 13 OR entity1 = 'geoId/15003')`) for CDC 500 StatVars. To prevent double-counting Oahu/Honolulu County population in years where 13-character Census Designated Places (`geoId/15XXXXX`) are published (2017–2022), `geoId/15003` is dynamically included via a SQL window check only when zero 13-character CDPs exist for Hawaii (`geoId/15`) in that `(StatVar, date, measurement_method)` partition (preserving coverage for 2013–2016 and county-only 2017 indicators).
+   - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%'`) for CDC 500 StatVars. To prevent double-counting Oahu/Honolulu County population in years where 13-character Census Designated Places (`geoId/15XXXXX`) are published (2017–2022), `geoId/15003` is explicitly filtered in the `WHERE` clause to include only years `<= '2016'` and `'2017'` county-only indicators (excluding the four 2017 blood pressure and cholesterol indicators that also have 13-character CDP data).
    - **City Cohort Populations**: Fetches city-level population counts (`value AS population`) for the corresponding demographic cohort StatVars.
 
 ### Aggregation Methodology
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 14f3282c0e..f879157d00 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -58,7 +58,7 @@
   GROUP BY cdc500, pop_statvar
 ),
 
-svo_percent_dedup AS (
+svo_percent AS (
   SELECT
     O.variable_measured AS statvar,
     O.entity1 AS observation_about,
@@ -76,7 +76,24 @@
   INNER JOIN cdc_sv
     ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
-    AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')
+    AND (
+      LENGTH(O.entity1) = 13
+      OR (
+        O.entity1 = 'geoId/15003'
+        AND (
+          O.date <= '2016'
+          OR (
+            O.date = '2017'
+            AND O.variable_measured NOT IN (
+        'Percent_Person_WithHighBloodPressure',
+        'Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication',
+        'Percent_Person_WithHighCholesterol',
+        'Percent_Person_ReceivedCholesterolScreening'
+            )
+          )
+        )
+      )
+    )
     AND O.variable_measured LIKE 'Percent_%'
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
@@ -84,28 +101,6 @@
   ) = 1
 ),
 
-svo_percent AS (
-  SELECT
-    statvar,
-    observation_about,
-    observation_date,
-    percent,
-    measurement_method,
-    pop_statvar
-  FROM svo_percent_dedup
-  QUALIFY LENGTH(observation_about) = 13
-    OR (
-      observation_about = 'geoId/15003'
-      AND COUNTIF(LENGTH(observation_about) = 13) OVER (
-        PARTITION BY
-          statvar,
-          SUBSTR(observation_about, 1, 8),
-          observation_date,
-          measurement_method
-      ) = 0
-    )
-),
-
 svo_count AS (
   SELECT
     O.variable_measured AS population_statvar,
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 665bc2725b..5b1cf63755 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -49,7 +49,10 @@ def test_query_constants(self):
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
         self.assertIn(
             "AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')", query)
-        self.assertIn("COUNTIF(LENGTH(observation_about) = 13) OVER", query)
+        self.assertIn("O.entity1 = 'geoId/15003'", query)
+        self.assertIn("O.date <= '2016'", query)
+        self.assertIn("O.date = '2017'", query)
+        self.assertNotIn("svo_percent_dedup", query)
         self.assertIn("REGEXP_CONTAINS", query)
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
         self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)

From d23ffa0cbf1ae26425bed84403d9778c9858d0da Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 15 Sep 2026 20:19:09 +0000
Subject: [PATCH 24/44] Address code review findings: add defensive SQL NULL
 checks, simplify 2017 Honolulu regex, and test SQL directly in DuckDB

---
 scripts/us_cdc/cdc500_state/process.py        |  17 +-
 scripts/us_cdc/cdc500_state/process_test.py   | 243 ++++++++++++++----
 .../cdc500_state/validation_config.json       |   9 +
 3 files changed, 207 insertions(+), 62 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index f879157d00..e9e9ebe328 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -84,17 +84,15 @@
           O.date <= '2016'
           OR (
             O.date = '2017'
-            AND O.variable_measured NOT IN (
-        'Percent_Person_WithHighBloodPressure',
-        'Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication',
-        'Percent_Person_WithHighCholesterol',
-        'Percent_Person_ReceivedCholesterolScreening'
+            AND NOT REGEXP_CONTAINS(
+              O.variable_measured, r'HighBloodPressure|Cholesterol'
             )
           )
         )
       )
     )
     AND O.variable_measured LIKE 'Percent_%'
+    AND SAFE_CAST(O.value AS FLOAT64) IS NOT NULL
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
     ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
@@ -120,16 +118,17 @@
     ON O.variable_measured = pop.pop_statvar
   WHERE O.entity1 LIKE 'geoId/%'
     AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')
+    AND SAFE_CAST(O.value AS FLOAT64) IS NOT NULL
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date
     ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
   ) = 1
 )
 
-SELECT 
-  p.statvar, 
+SELECT
+  p.statvar,
   SUBSTR(p.observation_about, 1, 8) AS observation_about,
-  p.observation_date, 
+  p.observation_date,
   COALESCE(
     CONCAT('dcAggregate/', p.measurement_method), 'dcAggregate'
   ) AS measurement_method,
@@ -143,6 +142,7 @@
   AND p.observation_date = c.observation_date
   AND p.pop_statvar = c.population_statvar
 GROUP BY 1, 2, 3, 4
+HAVING percent IS NOT NULL
 """
 
 
@@ -187,4 +187,3 @@ def main(argv):
 
 if __name__ == '__main__':
     app.run(main)
-
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 5b1cf63755..ad5dc89c3d 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -21,6 +21,7 @@
 
 from absl import flags
 from absl.testing import flagsaver
+import duckdb
 import pandas as pd
 
 from scripts.us_cdc.cdc500_state import process
@@ -28,6 +29,19 @@
 FLAGS = flags.FLAGS
 
 
+def _prepare_duckdb_query(sql: str) -> str:
+    """Adapts BigQuery SQL syntax in process.QUERY for in-memory DuckDB execution."""
+    adapted = sql.replace(
+        '`datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries`', 'TimeSeries')
+    adapted = adapted.replace(
+        '`datcom-store.spanner_dc_graph_prod_DEFAULT.Observation`',
+        'Observation')
+    adapted = adapted.replace('SAFE_CAST(', 'TRY_CAST(')
+    adapted = adapted.replace('AS FLOAT64)', 'AS DOUBLE)')
+    adapted = adapted.replace("r'", "'")
+    return adapted
+
+
 class CDC500StateProcessTest(unittest.TestCase):
     """Unit tests for CDC 500 State aggregation processing."""
 
@@ -46,20 +60,26 @@ def test_query_constants(self):
         self.assertIn("dc/base/CensusACS5YearSurvey", query)
         self.assertIn("SAFE_DIVIDE", query)
         self.assertIn("SAFE_CAST", query)
+        self.assertIn("SAFE_CAST(O.value AS FLOAT64) IS NOT NULL", query)
+        self.assertIn("HAVING percent IS NOT NULL", query)
         self.assertIn("SUBSTR(p.observation_about, 1, 8)", query)
         self.assertIn(
             "AND (LENGTH(O.entity1) = 13 OR O.entity1 = 'geoId/15003')", query)
         self.assertIn("O.entity1 = 'geoId/15003'", query)
         self.assertIn("O.date <= '2016'", query)
         self.assertIn("O.date = '2017'", query)
+        self.assertIn("HighBloodPressure|Cholesterol", query)
         self.assertNotIn("svo_percent_dedup", query)
         self.assertIn("REGEXP_CONTAINS", query)
         self.assertIn("QUALIFY ROW_NUMBER() OVER", query)
         self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)
-        self.assertIn("Percent_Person_50To74Years_Female_ReceivedMammography", query)
-        self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest", query)
+        self.assertIn("Percent_Person_50To74Years_Female_ReceivedMammography",
+                      query)
+        self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest",
+                      query)
         self.assertIn(
-            "Percent_Person_50To75Years_ReceivedColorectalCancerScreening", query)
+            "Percent_Person_50To75Years_ReceivedColorectalCancerScreening",
+            query)
         self.assertIn("'Count_Person_65OrMoreYears_Female'", query)
         self.assertIn("'Count_Person_65OrMoreYears_Male'", query)
         self.assertIn("'Count_Person_65OrMoreYears'", query)
@@ -72,27 +92,11 @@ def test_query_constants(self):
         self.assertNotIn("p.pop_statvar AS population_statvar", query)
 
     def test_demographic_cohort_regex_mapping(self):
-        """Verifies that representative StatVars match the intended demographic regex rules."""
-        female_pattern = r'65OrMoreYears.*Female|Female.*65OrMoreYears'
-        male_pattern = r'65OrMoreYears.*Male|Male.*65OrMoreYears'
-
-        self.assertIn(female_pattern, process.QUERY)
-        self.assertIn(male_pattern, process.QUERY)
-
-        # Helper mapping that mirrors the SQL CASE WHEN logic
-        def map_statvar(sv: str) -> str:
-            if re.search(female_pattern, sv):
-                return 'Count_Person_65OrMoreYears_Female'
-            elif re.search(male_pattern, sv):
-                return 'Count_Person_65OrMoreYears_Male'
-            elif '65OrMoreYears' in sv:
-                return 'Count_Person_65OrMoreYears'
-            elif '18To64Years' in sv:
-                return 'Count_Person_18To64Years'
-            elif '18OrMoreYears' in sv:
-                return 'Count_Person_18OrMoreYears'
-            else:
-                return 'Count_Person'
+        """Executes cdc_sv CTE from process.QUERY in DuckDB to verify cohort mapping."""
+        con = duckdb.connect(':memory:')
+        con.create_function(
+            'REGEXP_CONTAINS', lambda s, p: bool(re.search(p, s))
+            if s and p else False, [str, str], bool)
 
         test_cases = [
             ('Percent_Person_65OrMoreYears_Female_CorePreventiveServices',
@@ -105,38 +109,168 @@ def map_statvar(sv: str) -> str:
              'Count_Person_65OrMoreYears_Male'),
             ('Percent_Person_65OrMoreYears_CorePreventiveServices',
              'Count_Person_65OrMoreYears'),
-            ('Percent_Person_18To64Years_HealthInsurance', 'Count_Person_18To64Years'),
-            ('Percent_Person_18OrMoreYears_WithAnyDisability', 'Count_Person_18OrMoreYears'),
+            ('Percent_Person_18To64Years_HealthInsurance',
+             'Count_Person_18To64Years'),
+            ('Percent_Person_18OrMoreYears_WithAnyDisability',
+             'Count_Person_18OrMoreYears'),
             ('Percent_Person_18OrMoreYears_WithHighBloodPressure',
              'Count_Person_18OrMoreYears'),
             ('Percent_Person_WithArthritis', 'Count_Person'),
             ('Percent_Person_WithHighCholesterol', 'Count_Person'),
         ]
+        excluded_statvars = [
+            'Percent_Person_50To74Years_Female_ReceivedMammography',
+            'Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening',
+            'Percent_Person_21To65Years_Female_ReceivedPapSmearTest',
+            'Percent_Person_50To75Years_ReceivedColorectalCancerScreening',
+        ]
+
+        rows = [{
+            'variable_measured': sv,
+            'provenance': 'dc/base/CDC500'
+        } for sv, _ in test_cases]
+        rows.extend([{
+            'variable_measured': sv,
+            'provenance': 'dc/base/CDC500'
+        } for sv in excluded_statvars])
+        ts_df = pd.DataFrame(rows)
+        con.register('TimeSeries', ts_df)
 
-        for sv, expected in test_cases:
+        # Extract cdc_sv CTE from process.QUERY and execute against TimeSeries
+        adapted_sql = _prepare_duckdb_query(process.QUERY)
+        cdc_sv_sql = adapted_sql.split('svo_percent AS (')[0].rstrip().rstrip(
+            ',')
+        result_df = con.execute(
+            f"{cdc_sv_sql} SELECT cdc500, pop_statvar FROM cdc_sv").df()
+
+        result_map = dict(zip(result_df['cdc500'], result_df['pop_statvar']))
+        self.assertEqual(len(result_map), len(test_cases))
+        for sv, expected_cohort in test_cases:
             with self.subTest(statvar=sv):
-                self.assertEqual(map_statvar(sv), expected)
+                self.assertEqual(result_map.get(sv), expected_cohort)
+        for excluded_sv in excluded_statvars:
+            with self.subTest(excluded=excluded_sv):
+                self.assertNotIn(excluded_sv, result_map)
 
     def test_population_weighted_average_calculation(self):
-        """Verifies the population-weighted average calculation and city-to-state
-        FIPS aggregation."""
-        # Simulated city-level records for California (geoId/06)
-        city_records = pd.DataFrame({
-            'city_geoid': ['geoId/0644000', 'geoId/0666000', 'geoId/0667000'],
-            'city_percent': [20.0, 30.0, 40.0],
-            'city_pop': [10000, 20000, 70000]
-        })
-        city_records['state_geoid'] = city_records['city_geoid'].str.slice(0, 8)
-        self.assertTrue((city_records['state_geoid'] == 'geoId/06').all())
+        """Executes full process.QUERY in DuckDB to verify state weighted average."""
+        con = duckdb.connect(':memory:')
+        con.create_function(
+            'REGEXP_CONTAINS', lambda s, p: bool(re.search(p, s))
+            if s and p else False, [str, str], bool)
+        con.create_function(
+            'SAFE_DIVIDE', lambda a, b: float(a) / float(b)
+            if (a is not None and b) else None, [float, float], float)
 
-        # Formula: SUM(pop * percent) / SUM(pop)
-        total_weighted = (city_records['city_pop'] * city_records['city_percent']).sum()
-        total_pop = city_records['city_pop'].sum()
-        weighted_avg = total_weighted / total_pop
+        ts_df = pd.DataFrame([
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0644000',
+                'facet_id': 'f1',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0666000',
+                'facet_id': 'f1',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0667000',
+                'facet_id': 'f1',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0644000',
+                'facet_id': 'f2',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0666000',
+                'facet_id': 'f2',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0667000',
+                'facet_id': 'f2',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
+        ])
+        obs_df = pd.DataFrame([
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0644000',
+                'date': '2022',
+                'value': '20.0',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0666000',
+                'date': '2022',
+                'value': '30.0',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0667000',
+                'date': '2022',
+                'value': '40.0',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0644000',
+                'date': '2022',
+                'value': '10000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0666000',
+                'date': '2022',
+                'value': '20000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0667000',
+                'date': '2022',
+                'value': '70000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
+        ])
+        con.register('TimeSeries', ts_df)
+        con.register('Observation', obs_df)
+
+        sql = _prepare_duckdb_query(process.QUERY)
+        result_df = con.execute(sql).df()
 
         # Expected: (10000*20 + 20000*30 + 70000*40) / 100000 = 36.0
-        self.assertEqual(total_pop, 100000)
-        self.assertAlmostEqual(weighted_avg, 36.0, places=4)
+        self.assertEqual(len(result_df), 1)
+        self.assertEqual(result_df['statvar'].iloc[0],
+                         'Percent_Person_WithArthritis')
+        self.assertEqual(result_df['observation_about'].iloc[0], 'geoId/06')
+        self.assertEqual(result_df['observation_date'].iloc[0], '2022')
+        self.assertEqual(result_df['measurement_method'].iloc[0],
+                         'dcAggregate/CrudePrevalence')
+        self.assertAlmostEqual(result_df['percent'].iloc[0], 36.0, places=4)
 
     def test_run_process_success(self):
         """Tests successful query execution and atomic output writing."""
@@ -164,7 +298,8 @@ def test_run_process_success(self):
     def test_run_process_empty_dataframe_raises_runtime_error(self):
         """Tests that empty query results raise RuntimeError."""
         mock_client = mock.MagicMock()
-        mock_client.query.return_value.to_dataframe.return_value = pd.DataFrame()
+        mock_client.query.return_value.to_dataframe.return_value = pd.DataFrame(
+        )
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
             with self.assertRaisesRegex(RuntimeError,
@@ -209,9 +344,8 @@ def fake_to_csv(filepath, index=False):
         mock_client.query.return_value.to_dataframe.return_value = mock_df
         with tempfile.TemporaryDirectory() as tmp_dir:
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaisesRegex(
-                    RuntimeError,
-                    "was created empty or missing"):
+            with self.assertRaisesRegex(RuntimeError,
+                                        "was created empty or missing"):
                 process.run_process(mock_client, output_file)
             self.assertFalse(os.path.exists(output_file))
             self.assertFalse(os.path.exists(output_file + '.tmp'))
@@ -223,12 +357,15 @@ def test_main(self, mock_bq_client_cls, mock_run_process):
         mock_client_instance = mock.MagicMock()
         mock_bq_client_cls.return_value = mock_client_instance
         with tempfile.TemporaryDirectory() as tmp_dir:
-            with flagsaver.flagsaver(output_dir=tmp_dir, project='test-project'):
+            with flagsaver.flagsaver(output_dir=tmp_dir,
+                                     project='test-project'):
                 process.main([])
-                expected_output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-                mock_bq_client_cls.assert_called_once_with(project='test-project')
-                mock_run_process.assert_called_once_with(mock_client_instance,
-                                                         expected_output_file)
+                expected_output_file = os.path.join(tmp_dir,
+                                                    'CDC500State_Output.csv')
+                mock_bq_client_cls.assert_called_once_with(
+                    project='test-project')
+                mock_run_process.assert_called_once_with(
+                    mock_client_instance, expected_output_file)
 
 
 if __name__ == '__main__':
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 3876fcbde1..c9f137e9e4 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -35,6 +35,15 @@
                 "minimum": 50,
                 "maximum": 52
             }
+        },
+        {
+            "rule_id": "check_statvar_max_dates",
+            "description": "Verifies that MaxDate meets expected vintage freshness per StatVar cohort.",
+            "validator": "SQL_VALIDATOR",
+            "params": {
+                "query": "SELECT StatVar, TRY_CAST(MaxDate AS INTEGER) AS max_year, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
+                "condition": "max_year >= expected_year"
+            }
         }
     ]
 }

From 0585f034b2878fbd3dbbbf6e606ea6e8517f30a5 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 16 Sep 2026 05:35:49 +0000
Subject: [PATCH 25/44] Address review feedback: harden SQL NULL/empty checks,
 simplify query, update README, and expand DuckDB tests

---
 scripts/us_cdc/cdc500_state/README.md         |  12 ++
 scripts/us_cdc/cdc500_state/process.py        |   5 +-
 scripts/us_cdc/cdc500_state/process_test.py   | 126 +++++++++++++++++-
 .../cdc500_state/validation_config.json       |   4 +-
 4 files changed, 139 insertions(+), 8 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 6180c641f0..0f9a82c08a 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -10,6 +10,7 @@ Author: Padma Gundapaneni @padma-g
 2. [About the Import](#about-the-import)
     1. [Artifacts](#artifacts)
     2. [Import Procedure](#import-procedure)
+    3. [Troubleshooting](#troubleshooting)
 
 ## About the Dataset
 
@@ -67,6 +68,9 @@ The following age-bracketed cancer screening indicators are omitted from state-l
 #### Validation Config
 [`validation_config.json`](validation_config.json)
 
+#### Manifest
+[`manifest.json`](manifest.json)
+
 ### Import Procedure
 
 #### Prerequisites
@@ -109,3 +113,11 @@ $ python3 -m unittest scripts.us_cdc.cdc500_state.process_test
 
 This import is automated via Data Commons Import Automation and scheduled to run weekly via Cloud Batch every Monday at 01:00 UTC (`cron_schedule: "0 1 * * 1"` in `manifest.json`).
 
+- **Import Type**: Automated (weekly Cloud Batch cron: `0 1 * * 1`)
+- **Production GCS Path**: `gs://datcom-prod-imports/scripts/us_cdc/cdc500_state/CDC500_States/`
+- **Test GCS Path**: `gs://datcom-import-test/scripts/us_cdc/cdc500_state/CDC500_States/`
+
+### Troubleshooting
+
+- **`RuntimeError: BigQuery query returned 0 rows`**: Verify ADC authentication (`gcloud auth application-default login`) and ensure read permissions on `datcom-store.spanner_dc_graph_prod_DEFAULT`. Pass `--project=<gcp_project_id>` if running outside the default project.
+- **`check_statvar_max_dates` validation failure**: When CDC PLACES publishes a new release year, update the `CASE` statement in `validation_config.json` (`check_statvar_max_dates`) to reflect the new expected vintage years per cohort.
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index e9e9ebe328..caa5292e03 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -72,7 +72,6 @@
     AND O.entity1 = T.entity1
     AND O.facet_id = T.facet_id
     AND T.provenance = 'dc/base/CDC500'
-    AND T.variable_measured LIKE 'Percent_%'
   INNER JOIN cdc_sv
     ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
@@ -91,7 +90,6 @@
         )
       )
     )
-    AND O.variable_measured LIKE 'Percent_%'
     AND SAFE_CAST(O.value AS FLOAT64) IS NOT NULL
   QUALIFY ROW_NUMBER() OVER (
     PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
@@ -130,7 +128,7 @@
   SUBSTR(p.observation_about, 1, 8) AS observation_about,
   p.observation_date,
   COALESCE(
-    CONCAT('dcAggregate/', p.measurement_method), 'dcAggregate'
+    'dcAggregate/' || NULLIF(TRIM(p.measurement_method), ''), 'dcAggregate'
   ) AS measurement_method,
   SAFE_DIVIDE(
     SUM(SAFE_CAST(c.population AS FLOAT64) * SAFE_CAST(p.percent AS FLOAT64)),
@@ -179,6 +177,7 @@ def run_process(client: bigquery.Client, output_file: str) -> bool:
 
 
 def main(argv):
+    """Main entry point for the CDC 500 state aggregation script."""
     del argv  # Unused.
     client = bigquery.Client(project=_FLAGS.project)
     output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index ad5dc89c3d..cefa9bc406 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -138,8 +138,8 @@ def test_demographic_cohort_regex_mapping(self):
 
         # Extract cdc_sv CTE from process.QUERY and execute against TimeSeries
         adapted_sql = _prepare_duckdb_query(process.QUERY)
-        cdc_sv_sql = adapted_sql.split('svo_percent AS (')[0].rstrip().rstrip(
-            ',')
+        cdc_sv_sql = adapted_sql.split('svo_percent AS (',
+                                       maxsplit=1)[0].rstrip().rstrip(',')
         result_df = con.execute(
             f"{cdc_sv_sql} SELECT cdc500, pop_statvar FROM cdc_sv").df()
 
@@ -272,6 +272,126 @@ def test_population_weighted_average_calculation(self):
                          'dcAggregate/CrudePrevalence')
         self.assertAlmostEqual(result_df['percent'].iloc[0], 36.0, places=4)
 
+    def test_hawaii_honolulu_county_edge_cases(self):
+        """Tests Honolulu County (geoId/15003) filtering and empty measurement_method."""
+        con = duckdb.connect(':memory:')
+        con.create_function(
+            'REGEXP_CONTAINS', lambda s, p: bool(re.search(p, s))
+            if s and p else False, [str, str], bool)
+        con.create_function(
+            'SAFE_DIVIDE', lambda a, b: float(a) / float(b)
+            if (a is not None and b) else None, [float, float], float)
+
+        # Test 2016 (county included), 2017 non-BP (county included),
+        # 2017 BP (county excluded, 13-char CDP used), and empty measurement_method.
+        ts_df = pd.DataFrame([
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/15003',
+                'facet_id': 'f1',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': ''
+            },
+            {
+                'variable_measured': 'Percent_Person_WithHighBloodPressure',
+                'entity1': 'geoId/15003',
+                'facet_id': 'f1',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
+            {
+                'variable_measured': 'Percent_Person_WithHighBloodPressure',
+                'entity1': 'geoId/1571550',
+                'facet_id': 'f1',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/15003',
+                'facet_id': 'f2',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/1571550',
+                'facet_id': 'f2',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
+        ])
+        obs_df = pd.DataFrame([
+            # 2016: geoId/15003 should be INCLUDED (value 22.5)
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/15003',
+                'date': '2016',
+                'value': '22.5',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/15003',
+                'date': '2016',
+                'value': '950000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
+            # 2017 HighBloodPressure: geoId/15003 (99.0) must be EXCLUDED;
+            # 13-char CDP geoId/1571550 (31.2) must be INCLUDED.
+            {
+                'variable_measured': 'Percent_Person_WithHighBloodPressure',
+                'entity1': 'geoId/15003',
+                'date': '2017',
+                'value': '99.0',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Percent_Person_WithHighBloodPressure',
+                'entity1': 'geoId/1571550',
+                'date': '2017',
+                'value': '31.2',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/15003',
+                'date': '2017',
+                'value': '950000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/1571550',
+                'date': '2017',
+                'value': '350000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
+        ])
+        con.register('TimeSeries', ts_df)
+        con.register('Observation', obs_df)
+
+        sql = _prepare_duckdb_query(process.QUERY)
+        result_df = con.execute(sql).df().sort_values(
+            by=['observation_date', 'statvar']).reset_index(drop=True)
+
+        self.assertEqual(len(result_df), 2)
+        # 2016 row: geoId/15003 included, empty measurement_method -> 'dcAggregate'
+        self.assertEqual(result_df['observation_date'].iloc[0], '2016')
+        self.assertEqual(result_df['measurement_method'].iloc[0], 'dcAggregate')
+        self.assertAlmostEqual(result_df['percent'].iloc[0], 22.5, places=4)
+        # 2017 BP row: geoId/15003 excluded, only geoId/1571550 (31.2) included
+        self.assertEqual(result_df['observation_date'].iloc[1], '2017')
+        self.assertEqual(result_df['measurement_method'].iloc[1],
+                         'dcAggregate/CrudePrevalence')
+        self.assertAlmostEqual(result_df['percent'].iloc[1], 31.2, places=4)
+
     def test_run_process_success(self):
         """Tests successful query execution and atomic output writing."""
         mock_client = mock.MagicMock()
@@ -337,7 +457,7 @@ def test_run_process_empty_output_file_raises_runtime_error(self):
 
         def fake_to_csv(filepath, index=False):
             del index  # Unused.
-            with open(filepath, 'w'):
+            with open(filepath, 'w', encoding='utf-8'):
                 pass  # Create 0-byte file
 
         mock_df.to_csv.side_effect = fake_to_csv
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index c9f137e9e4..64233dce2d 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -38,11 +38,11 @@
         },
         {
             "rule_id": "check_statvar_max_dates",
-            "description": "Verifies that MaxDate meets expected vintage freshness per StatVar cohort.",
+            "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
             "validator": "SQL_VALIDATOR",
             "params": {
                 "query": "SELECT StatVar, TRY_CAST(MaxDate AS INTEGER) AS max_year, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
-                "condition": "max_year >= expected_year"
+                "condition": "max_year IS NOT NULL AND max_year >= expected_year"
             }
         }
     ]

From 5cf8218b68023009b7b39100517bceec4b11ad0e Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Wed, 16 Sep 2026 06:46:53 +0000
Subject: [PATCH 26/44] Remove redundant cleaned_csv from source_files in
 manifest.json

---
 scripts/us_cdc/cdc500_state/manifest.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index 3053c3c89b..6c648380d7 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -22,7 +22,6 @@
                 "disk": 20
             },
             "source_files": [
-                "CDC500State_Output/CDC500State_Output.csv",
                 "validation_config.json"
             ],
             "cron_schedule": "0 1 * * 1",

From eecd9cc26c1173e83d161771c83ecd3d65f040ea Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Fri, 18 Sep 2026 06:15:40 +0000
Subject: [PATCH 27/44] Add BigQuery timeout and project logging in process.py;
 expand DuckDB tests for 2018 Honolulu and QUALIFY deduplication

---
 scripts/us_cdc/cdc500_state/README.md       |   4 +-
 scripts/us_cdc/cdc500_state/process.py      |  16 ++-
 scripts/us_cdc/cdc500_state/process_test.py | 152 +++++++++++++++++++-
 3 files changed, 158 insertions(+), 14 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 0f9a82c08a..f52f858cbf 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -95,10 +95,10 @@ To specify a custom output directory:
 $ python3 scripts/us_cdc/cdc500_state/process.py --output_dir=/path/to/output
 ```
 
-To specify a GCP project for BigQuery jobs (if not set in ambient environment):
+To specify a GCP project and timeout (in seconds) for BigQuery jobs (if not set in ambient environment):
 
 ```bash
-$ python3 scripts/us_cdc/cdc500_state/process.py --project=my-gcp-project
+$ python3 scripts/us_cdc/cdc500_state/process.py --project=my-gcp-project --timeout=600
 ```
 
 #### Running Unit Tests
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index caa5292e03..76d45f540d 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -23,12 +23,15 @@
 _FLAGS = flags.FLAGS
 _MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
 _DEFAULT_OUTPUT_DIR = os.path.join(_MODULE_DIR, 'CDC500State_Output')
+DEFAULT_BQ_TIMEOUT_SECONDS = 600
 
 flags.DEFINE_string('output_dir', _DEFAULT_OUTPUT_DIR,
                     'Directory to write output CSV.')
 flags.DEFINE_string(
     'project', None,
     'GCP project ID for BigQuery. Defaults to ambient environment if omitted.')
+flags.DEFINE_integer('timeout', DEFAULT_BQ_TIMEOUT_SECONDS,
+                     'Timeout in seconds for BigQuery query and download.')
 
 QUERY = """
 WITH cdc_sv AS (
@@ -144,14 +147,17 @@
 """
 
 
-def run_process(client: bigquery.Client, output_file: str) -> bool:
+def run_process(client: bigquery.Client,
+                output_file: str,
+                timeout: int = DEFAULT_BQ_TIMEOUT_SECONDS) -> bool:
     """Executes the BigQuery query and writes the resulting DataFrame to output_file."""
-    logging.info("Running BigQuery aggregation query...")
-    query_job = client.query(QUERY)
+    logging.info("Running BigQuery aggregation query on project %s...",
+                 client.project)
+    query_job = client.query(QUERY, timeout=timeout)
     logging.info("BigQuery job started with ID: %s", query_job.job_id)
 
     logging.info("Fetching query results into dataframe...")
-    df = query_job.to_dataframe()
+    df = query_job.to_dataframe(timeout=timeout)
 
     if df.empty:
         logging.error("BigQuery query returned 0 rows.")
@@ -181,7 +187,7 @@ def main(argv):
     del argv  # Unused.
     client = bigquery.Client(project=_FLAGS.project)
     output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
-    run_process(client, output_file)
+    run_process(client, output_file, timeout=_FLAGS.timeout)
 
 
 if __name__ == '__main__':
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index cefa9bc406..1579de6188 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -273,7 +273,7 @@ def test_population_weighted_average_calculation(self):
         self.assertAlmostEqual(result_df['percent'].iloc[0], 36.0, places=4)
 
     def test_hawaii_honolulu_county_edge_cases(self):
-        """Tests Honolulu County (geoId/15003) filtering and empty measurement_method."""
+        """Tests Honolulu County (geoId/15003) filtering across 2016, 2017, 2018."""
         con = duckdb.connect(':memory:')
         con.create_function(
             'REGEXP_CONTAINS', lambda s, p: bool(re.search(p, s))
@@ -283,7 +283,8 @@ def test_hawaii_honolulu_county_edge_cases(self):
             if (a is not None and b) else None, [float, float], float)
 
         # Test 2016 (county included), 2017 non-BP (county included),
-        # 2017 BP (county excluded, 13-char CDP used), and empty measurement_method.
+        # 2017 BP (county excluded, 13-char CDP used),
+        # 2018 non-BP (county excluded, 13-char CDP used), and empty measurement_method.
         ts_df = pd.DataFrame([
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
@@ -292,6 +293,13 @@ def test_hawaii_honolulu_county_edge_cases(self):
                 'provenance': 'dc/base/CDC500',
                 'measurement_method': ''
             },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/1571550',
+                'facet_id': 'f1',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
             {
                 'variable_measured': 'Percent_Person_WithHighBloodPressure',
                 'entity1': 'geoId/15003',
@@ -373,6 +381,40 @@ def test_hawaii_honolulu_county_edge_cases(self):
                 'facet_id': 'f2',
                 'last_update_timestamp': 100
             },
+            # 2018 Arthritis: geoId/15003 (88.0) must be EXCLUDED;
+            # 13-char CDP geoId/1571550 (20.4) must be INCLUDED.
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/15003',
+                'date': '2018',
+                'value': '88.0',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/1571550',
+                'date': '2018',
+                'value': '20.4',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/15003',
+                'date': '2018',
+                'value': '950000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/1571550',
+                'date': '2018',
+                'value': '350000',
+                'facet_id': 'f2',
+                'last_update_timestamp': 100
+            },
         ])
         con.register('TimeSeries', ts_df)
         con.register('Observation', obs_df)
@@ -381,7 +423,7 @@ def test_hawaii_honolulu_county_edge_cases(self):
         result_df = con.execute(sql).df().sort_values(
             by=['observation_date', 'statvar']).reset_index(drop=True)
 
-        self.assertEqual(len(result_df), 2)
+        self.assertEqual(len(result_df), 3)
         # 2016 row: geoId/15003 included, empty measurement_method -> 'dcAggregate'
         self.assertEqual(result_df['observation_date'].iloc[0], '2016')
         self.assertEqual(result_df['measurement_method'].iloc[0], 'dcAggregate')
@@ -391,6 +433,97 @@ def test_hawaii_honolulu_county_edge_cases(self):
         self.assertEqual(result_df['measurement_method'].iloc[1],
                          'dcAggregate/CrudePrevalence')
         self.assertAlmostEqual(result_df['percent'].iloc[1], 31.2, places=4)
+        # 2018 Arthritis row: geoId/15003 excluded, only geoId/1571550 (20.4) included
+        self.assertEqual(result_df['observation_date'].iloc[2], '2018')
+        self.assertEqual(result_df['measurement_method'].iloc[2],
+                         'dcAggregate/CrudePrevalence')
+        self.assertAlmostEqual(result_df['percent'].iloc[2], 20.4, places=4)
+
+    def test_qualify_row_number_deduplication(self):
+        """Tests QUALIFY ROW_NUMBER() deduplication by timestamp and facet_id."""
+        con = duckdb.connect(':memory:')
+        con.create_function(
+            'REGEXP_CONTAINS', lambda s, p: bool(re.search(p, s))
+            if s and p else False, [str, str], bool)
+        con.create_function(
+            'SAFE_DIVIDE', lambda a, b: float(a) / float(b)
+            if (a is not None and b) else None, [float, float], float)
+
+        ts_df = pd.DataFrame([
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0644000',
+                'facet_id': 'f_old',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0644000',
+                'facet_id': 'f_new',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0644000',
+                'facet_id': 'f_low',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0644000',
+                'facet_id': 'f_high',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
+        ])
+        obs_df = pd.DataFrame([
+            # Competing percent observations: newer timestamp (200) must win over (100)
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0644000',
+                'date': '2022',
+                'value': '99.0',
+                'facet_id': 'f_old',
+                'last_update_timestamp': 100
+            },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0644000',
+                'date': '2022',
+                'value': '25.0',
+                'facet_id': 'f_new',
+                'last_update_timestamp': 200
+            },
+            # Competing count observations with identical timestamp (200):
+            # higher facet_id ('f_low' > 'f_high' alphabetically: 'f_low' wins)
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0644000',
+                'date': '2022',
+                'value': '10000',
+                'facet_id': 'f_high',
+                'last_update_timestamp': 200
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0644000',
+                'date': '2022',
+                'value': '50000',
+                'facet_id': 'f_low',
+                'last_update_timestamp': 200
+            },
+        ])
+        con.register('TimeSeries', ts_df)
+        con.register('Observation', obs_df)
+
+        sql = _prepare_duckdb_query(process.QUERY)
+        result_df = con.execute(sql).df()
+
+        self.assertEqual(len(result_df), 1)
+        self.assertAlmostEqual(result_df['percent'].iloc[0], 25.0, places=4)
 
     def test_run_process_success(self):
         """Tests successful query execution and atomic output writing."""
@@ -408,7 +541,10 @@ def test_run_process_success(self):
             output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
             result = process.run_process(mock_client, output_file)
             self.assertTrue(result)
-            mock_client.query.assert_called_once_with(process.QUERY)
+            mock_client.query.assert_called_once_with(
+                process.QUERY, timeout=process.DEFAULT_BQ_TIMEOUT_SECONDS)
+            mock_client.query.return_value.to_dataframe.assert_called_once_with(
+                timeout=process.DEFAULT_BQ_TIMEOUT_SECONDS)
             self.assertTrue(os.path.exists(output_file))
             self.assertFalse(os.path.exists(output_file + '.tmp'))
             saved_df = pd.read_csv(output_file)
@@ -478,14 +614,16 @@ def test_main(self, mock_bq_client_cls, mock_run_process):
         mock_bq_client_cls.return_value = mock_client_instance
         with tempfile.TemporaryDirectory() as tmp_dir:
             with flagsaver.flagsaver(output_dir=tmp_dir,
-                                     project='test-project'):
+                                     project='test-project',
+                                     timeout=300):
                 process.main([])
                 expected_output_file = os.path.join(tmp_dir,
                                                     'CDC500State_Output.csv')
                 mock_bq_client_cls.assert_called_once_with(
                     project='test-project')
-                mock_run_process.assert_called_once_with(
-                    mock_client_instance, expected_output_file)
+                mock_run_process.assert_called_once_with(mock_client_instance,
+                                                         expected_output_file,
+                                                         timeout=300)
 
 
 if __name__ == '__main__':

From cea521ca590c33c196d01a608768b0b25679a7ae Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Fri, 18 Sep 2026 11:11:57 +0000
Subject: [PATCH 28/44] Refine validation description and clarify CDC PLACES
 lineage in README

---
 scripts/us_cdc/cdc500_state/README.md              | 2 +-
 scripts/us_cdc/cdc500_state/validation_config.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index f52f858cbf..1d51fe7878 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -120,4 +120,4 @@ This import is automated via Data Commons Import Automation and scheduled to run
 ### Troubleshooting
 
 - **`RuntimeError: BigQuery query returned 0 rows`**: Verify ADC authentication (`gcloud auth application-default login`) and ensure read permissions on `datcom-store.spanner_dc_graph_prod_DEFAULT`. Pass `--project=<gcp_project_id>` if running outside the default project.
-- **`check_statvar_max_dates` validation failure**: When CDC PLACES publishes a new release year, update the `CASE` statement in `validation_config.json` (`check_statvar_max_dates`) to reflect the new expected vintage years per cohort.
+- **`check_statvar_max_dates` validation failure**: When CDC PLACES (the official successor program to CDC 500 Cities) publishes a new release year, update the `CASE` statement in `validation_config.json` (`check_statvar_max_dates`) to reflect the new expected vintage years per cohort.
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 64233dce2d..11195011af 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -29,7 +29,7 @@
         },
         {
             "rule_id": "check_num_places_state_count",
-            "description": "Checks that state-level observations cover all 50-52 US states.",
+            "description": "Checks that state observations cover all 50 US states plus DC (50-52 entities).",
             "validator": "NUM_PLACES_COUNT",
             "params": {
                 "minimum": 50,

From f900d1e3ec025d62fbf51a800cf3bed5d9e6a0d8 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Fri, 18 Sep 2026 12:52:49 +0000
Subject: [PATCH 29/44] Allow 100% deletion threshold for TMCF
 unit/scalingFactor migration and harden DuckDB test assertions

---
 scripts/us_cdc/cdc500_state/README.md         |  2 +
 scripts/us_cdc/cdc500_state/process_test.py   | 78 ++++++++++++++++---
 .../cdc500_state/validation_config.json       | 10 ++-
 3 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 1d51fe7878..0f50ac7404 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -17,6 +17,8 @@ Author: Padma Gundapaneni @padma-g
 ### Overview
 The state-level dataset calculates aggregated health indicator prevalence estimates for US states from the city-level CDC 500 Cities (`CDC500`) project data, weighted by corresponding Census ACS 5-Year population counts.
 
+CDC PLACES (Population Level Analysis and Community Estimates) is the official successor program to the original CDC 500 Cities project, expanding health estimates across all US counties, places, and census tracts. In Data Commons, city-level data from both the historical CDC 500 Cities project (2016–2019) and ongoing CDC PLACES releases (2020–present) are ingested under provenance `dc/base/CDC500`.
+
 ### Data Sources and Tables
 
 The aggregation script queries Google Cloud BigQuery graph tables in dataset `datcom-store.spanner_dc_graph_prod_DEFAULT`:
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 1579de6188..99af6437a1 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -347,6 +347,15 @@ def test_hawaii_honolulu_county_edge_cases(self):
                 'facet_id': 'f2',
                 'last_update_timestamp': 100
             },
+            # 2017 non-BP Arthritis: geoId/15003 (24.0) must be INCLUDED
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/15003',
+                'date': '2017',
+                'value': '24.0',
+                'facet_id': 'f1',
+                'last_update_timestamp': 100
+            },
             # 2017 HighBloodPressure: geoId/15003 (99.0) must be EXCLUDED;
             # 13-char CDP geoId/1571550 (31.2) must be INCLUDED.
             {
@@ -423,21 +432,33 @@ def test_hawaii_honolulu_county_edge_cases(self):
         result_df = con.execute(sql).df().sort_values(
             by=['observation_date', 'statvar']).reset_index(drop=True)
 
-        self.assertEqual(len(result_df), 3)
+        self.assertEqual(len(result_df), 4)
         # 2016 row: geoId/15003 included, empty measurement_method -> 'dcAggregate'
         self.assertEqual(result_df['observation_date'].iloc[0], '2016')
+        self.assertEqual(result_df['statvar'].iloc[0],
+                         'Percent_Person_WithArthritis')
         self.assertEqual(result_df['measurement_method'].iloc[0], 'dcAggregate')
         self.assertAlmostEqual(result_df['percent'].iloc[0], 22.5, places=4)
-        # 2017 BP row: geoId/15003 excluded, only geoId/1571550 (31.2) included
+        # 2017 non-BP row: geoId/15003 included
         self.assertEqual(result_df['observation_date'].iloc[1], '2017')
-        self.assertEqual(result_df['measurement_method'].iloc[1],
+        self.assertEqual(result_df['statvar'].iloc[1],
+                         'Percent_Person_WithArthritis')
+        self.assertEqual(result_df['measurement_method'].iloc[1], 'dcAggregate')
+        self.assertAlmostEqual(result_df['percent'].iloc[1], 24.0, places=4)
+        # 2017 BP row: geoId/15003 excluded, only geoId/1571550 (31.2) included
+        self.assertEqual(result_df['observation_date'].iloc[2], '2017')
+        self.assertEqual(result_df['statvar'].iloc[2],
+                         'Percent_Person_WithHighBloodPressure')
+        self.assertEqual(result_df['measurement_method'].iloc[2],
                          'dcAggregate/CrudePrevalence')
-        self.assertAlmostEqual(result_df['percent'].iloc[1], 31.2, places=4)
+        self.assertAlmostEqual(result_df['percent'].iloc[2], 31.2, places=4)
         # 2018 Arthritis row: geoId/15003 excluded, only geoId/1571550 (20.4) included
-        self.assertEqual(result_df['observation_date'].iloc[2], '2018')
-        self.assertEqual(result_df['measurement_method'].iloc[2],
+        self.assertEqual(result_df['observation_date'].iloc[3], '2018')
+        self.assertEqual(result_df['statvar'].iloc[3],
+                         'Percent_Person_WithArthritis')
+        self.assertEqual(result_df['measurement_method'].iloc[3],
                          'dcAggregate/CrudePrevalence')
-        self.assertAlmostEqual(result_df['percent'].iloc[2], 20.4, places=4)
+        self.assertAlmostEqual(result_df['percent'].iloc[3], 20.4, places=4)
 
     def test_qualify_row_number_deduplication(self):
         """Tests QUALIFY ROW_NUMBER() deduplication by timestamp and facet_id."""
@@ -464,6 +485,13 @@ def test_qualify_row_number_deduplication(self):
                 'provenance': 'dc/base/CDC500',
                 'measurement_method': 'CrudePrevalence'
             },
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0666000',
+                'facet_id': 'f_city2',
+                'provenance': 'dc/base/CDC500',
+                'measurement_method': 'CrudePrevalence'
+            },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0644000',
@@ -478,9 +506,17 @@ def test_qualify_row_number_deduplication(self):
                 'provenance': 'dc/base/CensusACS5YearSurvey',
                 'measurement_method': ''
             },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0666000',
+                'facet_id': 'f_city2_pop',
+                'provenance': 'dc/base/CensusACS5YearSurvey',
+                'measurement_method': ''
+            },
         ])
         obs_df = pd.DataFrame([
-            # Competing percent observations: newer timestamp (200) must win over (100)
+            # Competing percent observations for geoId/0644000:
+            # newer timestamp (200) must win over (100)
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/0644000',
@@ -497,8 +533,25 @@ def test_qualify_row_number_deduplication(self):
                 'facet_id': 'f_new',
                 'last_update_timestamp': 200
             },
-            # Competing count observations with identical timestamp (200):
-            # higher facet_id ('f_low' > 'f_high' alphabetically: 'f_low' wins)
+            # Second city in CA (geoId/0666000) with percent 10.0 and pop 10,000
+            {
+                'variable_measured': 'Percent_Person_WithArthritis',
+                'entity1': 'geoId/0666000',
+                'date': '2022',
+                'value': '10.0',
+                'facet_id': 'f_city2',
+                'last_update_timestamp': 200
+            },
+            {
+                'variable_measured': 'Count_Person',
+                'entity1': 'geoId/0666000',
+                'date': '2022',
+                'value': '10000',
+                'facet_id': 'f_city2_pop',
+                'last_update_timestamp': 200
+            },
+            # Competing count observations for geoId/0644000 with identical timestamp (200):
+            # higher facet_id ('f_low' > 'f_high' alphabetically: 'f_low' (50,000) wins)
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0644000',
@@ -523,7 +576,10 @@ def test_qualify_row_number_deduplication(self):
         result_df = con.execute(sql).df()
 
         self.assertEqual(len(result_df), 1)
-        self.assertAlmostEqual(result_df['percent'].iloc[0], 25.0, places=4)
+        # Choosing 'f_low' (50,000) over 'f_high' (10,000) yields weighted average:
+        # (25.0 * 50,000 + 10.0 * 10,000) / (50,000 + 10,000) = 1,350,000 / 60,000 = 22.5.
+        # If 'f_high' (10,000) had been chosen, average would be 17.5.
+        self.assertAlmostEqual(result_df['percent'].iloc[0], 22.5, places=4)
 
     def test_run_process_success(self):
         """Tests successful query execution and atomic output writing."""
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 11195011af..95e4925260 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -1,6 +1,14 @@
 {
     "schema_version": "1.0",
     "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Allows 100% deletion for one-time TMCF series key migration (adding unit & scalingFactor).",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 100
+            }
+        },
         {
             "rule_id": "check_max_value_percentage",
             "description": "Checks that all percentage StatVars do not exceed 100%.",
@@ -29,7 +37,7 @@
         },
         {
             "rule_id": "check_num_places_state_count",
-            "description": "Checks that state observations cover all 50 US states plus DC (50-52 entities).",
+            "description": "Checks state observations cover all 50 US states plus DC (50-52 entities).",
             "validator": "NUM_PLACES_COUNT",
             "params": {
                 "minimum": 50,

From 4cd11d99691ab608d79a3078d68c089a829e608a Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 21 Sep 2026 13:56:57 +0000
Subject: [PATCH 30/44] Address CRA findings: simplify regexes, add
 logging.fatal in main, remove 100% deletion threshold, and document rollout
 SOP

---
 scripts/us_cdc/cdc500_state/README.md          | 17 ++++++++++++++++-
 scripts/us_cdc/cdc500_state/process.py         |  9 ++++++---
 scripts/us_cdc/cdc500_state/process_test.py    | 18 ++++++++++++++----
 .../us_cdc/cdc500_state/validation_config.json |  8 --------
 4 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 0f50ac7404..3b3f61f318 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -10,7 +10,8 @@ Author: Padma Gundapaneni @padma-g
 2. [About the Import](#about-the-import)
     1. [Artifacts](#artifacts)
     2. [Import Procedure](#import-procedure)
-    3. [Troubleshooting](#troubleshooting)
+    3. [Production Rollout / Deployment SOP](#production-rollout--deployment-sop)
+    4. [Troubleshooting](#troubleshooting)
 
 ## About the Dataset
 
@@ -119,6 +120,20 @@ This import is automated via Data Commons Import Automation and scheduled to run
 - **Production GCS Path**: `gs://datcom-prod-imports/scripts/us_cdc/cdc500_state/CDC500_States/`
 - **Test GCS Path**: `gs://datcom-import-test/scripts/us_cdc/cdc500_state/CDC500_States/`
 
+### Production Rollout / Deployment SOP
+
+Adding `unit: Percent` and `scalingFactor: 100` to `cdc500_state.tmcf` alters the observation series identity key (`_GROUPBY_KEYS` in `import_differ.py`). Because historical production baselines (`latest_version.txt`) lack these fields, comparing new runs against the old baseline reports 100% deletions. To maintain strict 0% deletion safety (`threshold: 0`) in steady-state automated weekly crons without triggering false deletion alarms, follow this 2-run deployment workflow upon merging:
+
+1. **Step 1: Trigger Manual Production Run 1**:
+   Immediately upon merge to `master`, manually trigger a production Cloud Batch import run to generate Run 1 incorporating `unit: Percent` and `scalingFactor: 100`.
+2. **Step 2: Update Production Baseline**:
+   Manually point `latest_version.txt` in GCS to Run 1's version directory:
+   ```bash
+   echo "<RUN_1_TIMESTAMP_VERSION>" | gcloud storage cp - gs://datcom-prod-imports/scripts/us_cdc/cdc500_state/CDC500_States/latest_version.txt
+   ```
+3. **Step 3: Verify Steady-State Differ**:
+   Trigger Run 2 (or await the scheduled Monday cron at `01:00 UTC`). Verify that `import_differ` compares Run 2 against Run 1, passing `check_deleted_records_percent` with 0% deletions (`0.0%`) under the strict default threshold (`threshold: 0`).
+
 ### Troubleshooting
 
 - **`RuntimeError: BigQuery query returned 0 rows`**: Verify ADC authentication (`gcloud auth application-default login`) and ensure read permissions on `datcom-store.spanner_dc_graph_prod_DEFAULT`. Pass `--project=<gcp_project_id>` if running outside the default project.
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 76d45f540d..be84f494e7 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -39,10 +39,10 @@
     variable_measured AS cdc500,
     CASE
       WHEN REGEXP_CONTAINS(
-        variable_measured, r'65OrMoreYears.*Female|Female.*65OrMoreYears'
+        variable_measured, r'65OrMoreYears.*Female'
       ) THEN 'Count_Person_65OrMoreYears_Female'
       WHEN REGEXP_CONTAINS(
-        variable_measured, r'65OrMoreYears.*Male|Male.*65OrMoreYears'
+        variable_measured, r'65OrMoreYears.*Male'
       ) THEN 'Count_Person_65OrMoreYears_Male'
       WHEN variable_measured LIKE '%65OrMoreYears%' THEN 'Count_Person_65OrMoreYears'
       WHEN variable_measured LIKE '%18To64Years%' THEN 'Count_Person_18To64Years'
@@ -187,7 +187,10 @@ def main(argv):
     del argv  # Unused.
     client = bigquery.Client(project=_FLAGS.project)
     output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
-    run_process(client, output_file, timeout=_FLAGS.timeout)
+    try:
+        run_process(client, output_file, timeout=_FLAGS.timeout)
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logging.fatal("CDC 500 state aggregation failed: %s", e, exc_info=True)
 
 
 if __name__ == '__main__':
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 99af6437a1..36de74d3c0 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -101,12 +101,8 @@ def test_demographic_cohort_regex_mapping(self):
         test_cases = [
             ('Percent_Person_65OrMoreYears_Female_CorePreventiveServices',
              'Count_Person_65OrMoreYears_Female'),
-            ('Percent_Person_Female_65OrMoreYears_CorePreventiveServices',
-             'Count_Person_65OrMoreYears_Female'),
             ('Percent_Person_65OrMoreYears_Male_CorePreventiveServices',
              'Count_Person_65OrMoreYears_Male'),
-            ('Percent_Person_Male_65OrMoreYears_CorePreventiveServices',
-             'Count_Person_65OrMoreYears_Male'),
             ('Percent_Person_65OrMoreYears_CorePreventiveServices',
              'Count_Person_65OrMoreYears'),
             ('Percent_Person_18To64Years_HealthInsurance',
@@ -681,6 +677,20 @@ def test_main(self, mock_bq_client_cls, mock_run_process):
                                                          expected_output_file,
                                                          timeout=300)
 
+    @mock.patch('scripts.us_cdc.cdc500_state.process.logging.fatal')
+    @mock.patch('scripts.us_cdc.cdc500_state.process.run_process')
+    @mock.patch('google.cloud.bigquery.Client')
+    def test_main_error_logs_fatal(self, mock_bq_client_cls, mock_run_process,
+                                   mock_logging_fatal):
+        """Tests process.main catches unhandled exceptions and logs via logging.fatal."""
+        del mock_bq_client_cls  # Unused.
+        mock_run_process.side_effect = RuntimeError(
+            "BigQuery query returned 0 rows.")
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            with flagsaver.flagsaver(output_dir=tmp_dir):
+                process.main([])
+                mock_logging_fatal.assert_called_once()
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 95e4925260..9bffedc90d 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -1,14 +1,6 @@
 {
     "schema_version": "1.0",
     "rules": [
-        {
-            "rule_id": "check_deleted_records_percent",
-            "description": "Allows 100% deletion for one-time TMCF series key migration (adding unit & scalingFactor).",
-            "validator": "DELETED_RECORDS_PERCENT",
-            "params": {
-                "threshold": 100
-            }
-        },
         {
             "rule_id": "check_max_value_percentage",
             "description": "Checks that all percentage StatVars do not exceed 100%.",

From aca4c39e1eb35a878ee2af55c7adf2c66bf1318c Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 21 Sep 2026 14:44:42 +0000
Subject: [PATCH 31/44] Address P3 review polish: move client init inside try
 in main, add cervical screening assertion in tests, and fix JSON line length

---
 scripts/us_cdc/cdc500_state/process.py            |  2 +-
 scripts/us_cdc/cdc500_state/process_test.py       | 15 +++++++++++++++
 .../us_cdc/cdc500_state/validation_config.json    |  2 +-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index be84f494e7..f88bb13039 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -185,9 +185,9 @@ def run_process(client: bigquery.Client,
 def main(argv):
     """Main entry point for the CDC 500 state aggregation script."""
     del argv  # Unused.
-    client = bigquery.Client(project=_FLAGS.project)
     output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
     try:
+        client = bigquery.Client(project=_FLAGS.project)
         run_process(client, output_file, timeout=_FLAGS.timeout)
     except Exception as e:  # pylint: disable=broad-exception-caught
         logging.fatal("CDC 500 state aggregation failed: %s", e, exc_info=True)
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 36de74d3c0..6746250ead 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -75,6 +75,9 @@ def test_query_constants(self):
         self.assertIn("O.last_update_timestamp DESC, O.facet_id DESC", query)
         self.assertIn("Percent_Person_50To74Years_Female_ReceivedMammography",
                       query)
+        self.assertIn(
+            "Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening",
+            query)
         self.assertIn("Percent_Person_21To65Years_Female_ReceivedPapSmearTest",
                       query)
         self.assertIn(
@@ -691,6 +694,18 @@ def test_main_error_logs_fatal(self, mock_bq_client_cls, mock_run_process,
                 process.main([])
                 mock_logging_fatal.assert_called_once()
 
+    @mock.patch('scripts.us_cdc.cdc500_state.process.logging.fatal')
+    @mock.patch('google.cloud.bigquery.Client')
+    def test_main_client_init_error_logs_fatal(self, mock_bq_client_cls,
+                                               mock_logging_fatal):
+        """Tests process.main catches client init errors and logs via logging.fatal."""
+        mock_bq_client_cls.side_effect = RuntimeError(
+            "DefaultCredentialsError: Could not automatically determine credentials.")
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            with flagsaver.flagsaver(output_dir=tmp_dir):
+                process.main([])
+                mock_logging_fatal.assert_called_once()
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 9bffedc90d..c63bd8862b 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -29,7 +29,7 @@
         },
         {
             "rule_id": "check_num_places_state_count",
-            "description": "Checks state observations cover all 50 US states plus DC (50-52 entities).",
+            "description": "Checks observations cover 50 US states plus DC (50-52 entities).",
             "validator": "NUM_PLACES_COUNT",
             "params": {
                 "minimum": 50,

From ca314c308731c5bbd6d1b8cf5d329daf51b82b54 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 21 Sep 2026 15:46:11 +0000
Subject: [PATCH 32/44] Shorten SQL_VALIDATOR query in validation_config.json
 to satisfy 100-char line limit

---
 scripts/us_cdc/cdc500_state/validation_config.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index c63bd8862b..e30dee1221 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -41,8 +41,8 @@
             "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
             "validator": "SQL_VALIDATOR",
             "params": {
-                "query": "SELECT StatVar, TRY_CAST(MaxDate AS INTEGER) AS max_year, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
-                "condition": "max_year IS NOT NULL AND max_year >= expected_year"
+                "query": "SELECT MIN(MaxDate) AS min_d, MAX(MaxDate) AS max_d FROM stats",
+                "condition": "min_d >= '2016' AND max_d >= '2022'"
             }
         }
     ]

From 98149ab6df8fbad3ad00ed400cf620fe579846aa Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 21 Sep 2026 19:05:59 +0000
Subject: [PATCH 33/44] Address review findings: restore cohort-specific date
 freshness check, fix yapf formatting, and add Honolulu state assertion

---
 scripts/us_cdc/cdc500_state/README.md              | 1 +
 scripts/us_cdc/cdc500_state/process_test.py        | 7 ++++++-
 scripts/us_cdc/cdc500_state/validation_config.json | 4 ++--
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 3b3f61f318..3d6aefe894 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -126,6 +126,7 @@ Adding `unit: Percent` and `scalingFactor: 100` to `cdc500_state.tmcf` alters th
 
 1. **Step 1: Trigger Manual Production Run 1**:
    Immediately upon merge to `master`, manually trigger a production Cloud Batch import run to generate Run 1 incorporating `unit: Percent` and `scalingFactor: 100`.
+   *(Note: Run 1 is expected to fail validation at the differ step because `check_deleted_records_percent` reports 100% deletions against the pre-migration baseline lacking `unit`/`scalingFactor`. The output dataset is successfully generated; proceed to Step 2 to promote Run 1 as the new baseline.)*
 2. **Step 2: Update Production Baseline**:
    Manually point `latest_version.txt` in GCS to Run 1's version directory:
    ```bash
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 6746250ead..929cea53a5 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -433,18 +433,21 @@ def test_hawaii_honolulu_county_edge_cases(self):
 
         self.assertEqual(len(result_df), 4)
         # 2016 row: geoId/15003 included, empty measurement_method -> 'dcAggregate'
+        self.assertEqual(result_df['observation_about'].iloc[0], 'geoId/15')
         self.assertEqual(result_df['observation_date'].iloc[0], '2016')
         self.assertEqual(result_df['statvar'].iloc[0],
                          'Percent_Person_WithArthritis')
         self.assertEqual(result_df['measurement_method'].iloc[0], 'dcAggregate')
         self.assertAlmostEqual(result_df['percent'].iloc[0], 22.5, places=4)
         # 2017 non-BP row: geoId/15003 included
+        self.assertEqual(result_df['observation_about'].iloc[1], 'geoId/15')
         self.assertEqual(result_df['observation_date'].iloc[1], '2017')
         self.assertEqual(result_df['statvar'].iloc[1],
                          'Percent_Person_WithArthritis')
         self.assertEqual(result_df['measurement_method'].iloc[1], 'dcAggregate')
         self.assertAlmostEqual(result_df['percent'].iloc[1], 24.0, places=4)
         # 2017 BP row: geoId/15003 excluded, only geoId/1571550 (31.2) included
+        self.assertEqual(result_df['observation_about'].iloc[2], 'geoId/15')
         self.assertEqual(result_df['observation_date'].iloc[2], '2017')
         self.assertEqual(result_df['statvar'].iloc[2],
                          'Percent_Person_WithHighBloodPressure')
@@ -452,6 +455,7 @@ def test_hawaii_honolulu_county_edge_cases(self):
                          'dcAggregate/CrudePrevalence')
         self.assertAlmostEqual(result_df['percent'].iloc[2], 31.2, places=4)
         # 2018 Arthritis row: geoId/15003 excluded, only geoId/1571550 (20.4) included
+        self.assertEqual(result_df['observation_about'].iloc[3], 'geoId/15')
         self.assertEqual(result_df['observation_date'].iloc[3], '2018')
         self.assertEqual(result_df['statvar'].iloc[3],
                          'Percent_Person_WithArthritis')
@@ -700,7 +704,8 @@ def test_main_client_init_error_logs_fatal(self, mock_bq_client_cls,
                                                mock_logging_fatal):
         """Tests process.main catches client init errors and logs via logging.fatal."""
         mock_bq_client_cls.side_effect = RuntimeError(
-            "DefaultCredentialsError: Could not automatically determine credentials.")
+            "DefaultCredentialsError: Could not automatically determine credentials."
+        )
         with tempfile.TemporaryDirectory() as tmp_dir:
             with flagsaver.flagsaver(output_dir=tmp_dir):
                 process.main([])
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index e30dee1221..c63bd8862b 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -41,8 +41,8 @@
             "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
             "validator": "SQL_VALIDATOR",
             "params": {
-                "query": "SELECT MIN(MaxDate) AS min_d, MAX(MaxDate) AS max_d FROM stats",
-                "condition": "min_d >= '2016' AND max_d >= '2022'"
+                "query": "SELECT StatVar, TRY_CAST(MaxDate AS INTEGER) AS max_year, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
+                "condition": "max_year IS NOT NULL AND max_year >= expected_year"
             }
         }
     ]

From 5ffb554dec56f74f78e65a9a7d6bfc6792f9bb16 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 21 Sep 2026 19:57:12 +0000
Subject: [PATCH 34/44] Enhance date parsing robustness in validation_config
 and document direct test path in README

---
 scripts/us_cdc/cdc500_state/README.md              | 2 ++
 scripts/us_cdc/cdc500_state/validation_config.json | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 3d6aefe894..0e194a268c 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -109,6 +109,8 @@ $ python3 scripts/us_cdc/cdc500_state/process.py --project=my-gcp-project --time
 Run the test suite using Python's `unittest` runner from the repository root:
 
 ```bash
+$ python3 -m unittest scripts/us_cdc/cdc500_state/process_test.py
+# or via module path:
 $ python3 -m unittest scripts.us_cdc.cdc500_state.process_test
 ```
 
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index c63bd8862b..a391438995 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -41,7 +41,7 @@
             "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
             "validator": "SQL_VALIDATOR",
             "params": {
-                "query": "SELECT StatVar, TRY_CAST(MaxDate AS INTEGER) AS max_year, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
+                "query": "SELECT StatVar, TRY_CAST(SUBSTR(MaxDate, 1, 4) AS INTEGER) AS max_year, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
                 "condition": "max_year IS NOT NULL AND max_year >= expected_year"
             }
         }

From 4b0ddfef3adea55b5d87858b366db3cec0443a41 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 22 Sep 2026 08:05:50 +0000
Subject: [PATCH 35/44] Address review findings from GPaste 4586740472479744:
 bump disk limit to 30GB, add non-empty guard to SQL validator, and clarify
 legacy excluded indicators

---
 scripts/us_cdc/cdc500_state/README.md              | 2 +-
 scripts/us_cdc/cdc500_state/manifest.json          | 2 +-
 scripts/us_cdc/cdc500_state/validation_config.json | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 0e194a268c..2881c377d6 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -53,7 +53,7 @@ The following age-bracketed cancer screening indicators are omitted from state-l
 - `Percent_Person_21To65Years_Female_ReceivedPapSmearTest`
 - `Percent_Person_50To75Years_ReceivedColorectalCancerScreening`
 
-**Rationale**: The Census ACS 5-Year Survey does not publish single composite population StatVars for these non-standard multi-year age brackets (`50To74Years`, `21To65Years`, `50To75Years`). Rather than applying arbitrary proxy weights or risking silent row omission, these indicators are explicitly excluded from state aggregation.
+**Rationale**: The Census ACS 5-Year Survey does not publish single composite population StatVars for these non-standard multi-year age brackets (`50To74Years`, `21To65Years`, `50To75Years`). In the legacy query, these indicators were already silently dropped during the inner join with ACS population series. Rather than applying arbitrary proxy weights, these indicators are explicitly excluded in the query CTE, preserving full parity with the legacy pipeline while preventing unnecessary processing.
 
 ## About the Import
 
diff --git a/scripts/us_cdc/cdc500_state/manifest.json b/scripts/us_cdc/cdc500_state/manifest.json
index 6c648380d7..a1a8369f20 100644
--- a/scripts/us_cdc/cdc500_state/manifest.json
+++ b/scripts/us_cdc/cdc500_state/manifest.json
@@ -19,7 +19,7 @@
             "resource_limits": {
                 "cpu": 2,
                 "memory": 8,
-                "disk": 20
+                "disk": 30
             },
             "source_files": [
                 "validation_config.json"
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index a391438995..fd31ca5dbe 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -41,8 +41,8 @@
             "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
             "validator": "SQL_VALIDATOR",
             "params": {
-                "query": "SELECT StatVar, TRY_CAST(SUBSTR(MaxDate, 1, 4) AS INTEGER) AS max_year, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
-                "condition": "max_year IS NOT NULL AND max_year >= expected_year"
+                "query": "SELECT StatVar, TRY_CAST(SUBSTR(MaxDate, 1, 4) AS INTEGER) AS max_year,\nCOUNT(*) OVER () AS total_svs,\nCASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016\nWHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020\nWHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021\nELSE 2022 END AS expected_year FROM stats",
+                "condition": "max_year IS NOT NULL AND max_year >= expected_year AND total_svs > 0"
             }
         }
     ]

From 2b76416227f80f2becf469f3b91448519413f247 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 22 Sep 2026 14:58:53 +0000
Subject: [PATCH 36/44] Address review findings: format SQL query in
 validation_config and clean up mock helper in process_test

---
 scripts/us_cdc/cdc500_state/process_test.py        | 4 ++--
 scripts/us_cdc/cdc500_state/validation_config.json | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 929cea53a5..f2bfccdd81 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -650,8 +650,8 @@ def test_run_process_empty_output_file_raises_runtime_error(self):
         mock_df.empty = False
         mock_df.__len__.return_value = 1
 
-        def fake_to_csv(filepath, index=False):
-            del index  # Unused.
+        def fake_to_csv(filepath, *args, **kwargs):
+            del args, kwargs  # Unused.
             with open(filepath, 'w', encoding='utf-8'):
                 pass  # Create 0-byte file
 
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index fd31ca5dbe..78828715f8 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -41,7 +41,7 @@
             "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
             "validator": "SQL_VALIDATOR",
             "params": {
-                "query": "SELECT StatVar, TRY_CAST(SUBSTR(MaxDate, 1, 4) AS INTEGER) AS max_year,\nCOUNT(*) OVER () AS total_svs,\nCASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016\nWHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020\nWHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021\nELSE 2022 END AS expected_year FROM stats",
+                "query": "SELECT StatVar, TRY_CAST(SUBSTR(MaxDate, 1, 4) AS INTEGER) AS max_year, COUNT(*) OVER () AS total_svs, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
                 "condition": "max_year IS NOT NULL AND max_year >= expected_year AND total_svs > 0"
             }
         }

From 98e36e949fbd91e28245748e95247d3d2356caaf Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 22 Sep 2026 15:26:24 +0000
Subject: [PATCH 37/44] Cast MaxDate to VARCHAR before SUBSTR in
 validation_config.json SQL validator

---
 scripts/us_cdc/cdc500_state/validation_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 78828715f8..9ce2bcff5f 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -41,7 +41,7 @@
             "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
             "validator": "SQL_VALIDATOR",
             "params": {
-                "query": "SELECT StatVar, TRY_CAST(SUBSTR(MaxDate, 1, 4) AS INTEGER) AS max_year, COUNT(*) OVER () AS total_svs, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
+                "query": "SELECT StatVar, TRY_CAST(SUBSTR(CAST(MaxDate AS VARCHAR), 1, 4) AS INTEGER) AS max_year, COUNT(*) OVER () AS total_svs, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
                 "condition": "max_year IS NOT NULL AND max_year >= expected_year AND total_svs > 0"
             }
         }

From e1f29f7e3350fc8c2b68f36fa3a0c52a95c76d4e Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 5 Oct 2026 07:12:53 +0000
Subject: [PATCH 38/44] Use spanner_dc_graph_prod_snap_latest snapshot dataset
 in CDC 500 state query

- Update BigQuery dataset from spanner_dc_graph_prod_DEFAULT to spanner_dc_graph_prod_snap_latest in process.py to align with Mixer snapshot consistency.
- Update table replacement and query constant assertions in process_test.py.
- Update dataset documentation and troubleshooting guidance in README.md.
---
 scripts/us_cdc/cdc500_state/README.md       |  6 +++---
 scripts/us_cdc/cdc500_state/process.py      | 10 +++++-----
 scripts/us_cdc/cdc500_state/process_test.py |  9 +++++----
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 2881c377d6..fae96339dc 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -22,7 +22,7 @@ CDC PLACES (Population Level Analysis and Community Estimates) is the official s
 
 ### Data Sources and Tables
 
-The aggregation script queries Google Cloud BigQuery graph tables in dataset `datcom-store.spanner_dc_graph_prod_DEFAULT`:
+The aggregation script queries Google Cloud BigQuery graph tables in dataset `datcom-store.spanner_dc_graph_prod_snap_latest`:
 
 1. **`TimeSeries`**:
    - **CDC 500 Series**: Identifies CDC 500 Statistical Variables (`provenance = 'dc/base/CDC500'` and `variable_measured LIKE 'Percent_%'`) and extracts their measurement methods (`measurement_method`). It maps each percentage health metric to its appropriate denominator demographic cohort StatVar (e.g., `Count_Person_18OrMoreYears`, `Count_Person_18To64Years`, `Count_Person_65OrMoreYears`, `Count_Person`, etc.).
@@ -78,7 +78,7 @@ The following age-bracketed cancer screening indicators are omitted from state-l
 
 #### Prerequisites
 
-Ensure Google Cloud authentication is configured with access to BigQuery dataset `datcom-store.spanner_dc_graph_prod_DEFAULT`:
+Ensure Google Cloud authentication is configured with access to BigQuery dataset `datcom-store.spanner_dc_graph_prod_snap_latest`:
 
 ```bash
 $ gcloud auth application-default login
@@ -139,5 +139,5 @@ Adding `unit: Percent` and `scalingFactor: 100` to `cdc500_state.tmcf` alters th
 
 ### Troubleshooting
 
-- **`RuntimeError: BigQuery query returned 0 rows`**: Verify ADC authentication (`gcloud auth application-default login`) and ensure read permissions on `datcom-store.spanner_dc_graph_prod_DEFAULT`. Pass `--project=<gcp_project_id>` if running outside the default project.
+- **`RuntimeError: BigQuery query returned 0 rows`**: Verify ADC authentication (`gcloud auth application-default login`) and ensure read permissions on `datcom-store.spanner_dc_graph_prod_snap_latest`. Pass `--project=<gcp_project_id>` if running outside the default project.
 - **`check_statvar_max_dates` validation failure**: When CDC PLACES (the official successor program to CDC 500 Cities) publishes a new release year, update the `CASE` statement in `validation_config.json` (`check_statvar_max_dates`) to reflect the new expected vintage years per cohort.
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index f88bb13039..8ed1beb6e3 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -49,7 +49,7 @@
       WHEN variable_measured LIKE '%18OrMoreYears%' THEN 'Count_Person_18OrMoreYears'
       ELSE 'Count_Person'
     END AS pop_statvar
-  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries`
+  FROM `datcom-store.spanner_dc_graph_prod_snap_latest.TimeSeries`
   WHERE provenance = 'dc/base/CDC500'
     AND variable_measured LIKE 'Percent_%'
     AND variable_measured NOT IN (
@@ -69,8 +69,8 @@
     O.value AS percent,
     T.measurement_method AS measurement_method,
     cdc_sv.pop_statvar
-  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation` AS O
-  INNER JOIN `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries` AS T
+  FROM `datcom-store.spanner_dc_graph_prod_snap_latest.Observation` AS O
+  INNER JOIN `datcom-store.spanner_dc_graph_prod_snap_latest.TimeSeries` AS T
     ON O.variable_measured = T.variable_measured
     AND O.entity1 = T.entity1
     AND O.facet_id = T.facet_id
@@ -106,8 +106,8 @@
     O.entity1 AS observation_about,
     O.date AS observation_date,
     O.value AS population
-  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation` AS O
-  INNER JOIN `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries` AS T
+  FROM `datcom-store.spanner_dc_graph_prod_snap_latest.Observation` AS O
+  INNER JOIN `datcom-store.spanner_dc_graph_prod_snap_latest.TimeSeries` AS T
     ON O.variable_measured = T.variable_measured
     AND O.entity1 = T.entity1
     AND O.facet_id = T.facet_id
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index f2bfccdd81..b84d08e46f 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -32,9 +32,10 @@
 def _prepare_duckdb_query(sql: str) -> str:
     """Adapts BigQuery SQL syntax in process.QUERY for in-memory DuckDB execution."""
     adapted = sql.replace(
-        '`datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries`', 'TimeSeries')
+        '`datcom-store.spanner_dc_graph_prod_snap_latest.TimeSeries`',
+        'TimeSeries')
     adapted = adapted.replace(
-        '`datcom-store.spanner_dc_graph_prod_DEFAULT.Observation`',
+        '`datcom-store.spanner_dc_graph_prod_snap_latest.Observation`',
         'Observation')
     adapted = adapted.replace('SAFE_CAST(', 'TRY_CAST(')
     adapted = adapted.replace('AS FLOAT64)', 'AS DOUBLE)')
@@ -54,8 +55,8 @@ def setUpClass(cls):
     def test_query_constants(self):
         """Verifies key SQL clauses and excluded variables in process.QUERY."""
         query = process.QUERY
-        self.assertIn("spanner_dc_graph_prod_DEFAULT.TimeSeries", query)
-        self.assertIn("spanner_dc_graph_prod_DEFAULT.Observation", query)
+        self.assertIn("spanner_dc_graph_prod_snap_latest.TimeSeries", query)
+        self.assertIn("spanner_dc_graph_prod_snap_latest.Observation", query)
         self.assertIn("dc/base/CDC500", query)
         self.assertIn("dc/base/CensusACS5YearSurvey", query)
         self.assertIn("SAFE_DIVIDE", query)

From 9bd3f31b05919b47a02693cd8a992672510f8a27 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Tue, 6 Oct 2026 03:05:55 +0000
Subject: [PATCH 39/44] Address review findings: eliminate manual baseline
 copy, validate atomic write completeness, and use ValueError

- README.md: Remove manual latest_version.txt copy SOP per b/548893664 and CHK-9.3.
- process.py: Validate written record count against DataFrame length before atomic rename (CHK-2.2).
- process.py: Centralize OUTPUT_FILENAME constant and raise ValueError instead of RuntimeError for empty/truncated data (CHK-1.5, CHK-3.1).
- process_test.py: Update test assertions for ValueError, use OUTPUT_FILENAME, and add unit test for truncated file serialization.
---
 scripts/us_cdc/cdc500_state/README.md       | 13 +-----
 scripts/us_cdc/cdc500_state/process.py      | 15 +++++--
 scripts/us_cdc/cdc500_state/process_test.py | 47 +++++++++++++++------
 3 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index fae96339dc..488b4e251b 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -124,18 +124,7 @@ This import is automated via Data Commons Import Automation and scheduled to run
 
 ### Production Rollout / Deployment SOP
 
-Adding `unit: Percent` and `scalingFactor: 100` to `cdc500_state.tmcf` alters the observation series identity key (`_GROUPBY_KEYS` in `import_differ.py`). Because historical production baselines (`latest_version.txt`) lack these fields, comparing new runs against the old baseline reports 100% deletions. To maintain strict 0% deletion safety (`threshold: 0`) in steady-state automated weekly crons without triggering false deletion alarms, follow this 2-run deployment workflow upon merging:
-
-1. **Step 1: Trigger Manual Production Run 1**:
-   Immediately upon merge to `master`, manually trigger a production Cloud Batch import run to generate Run 1 incorporating `unit: Percent` and `scalingFactor: 100`.
-   *(Note: Run 1 is expected to fail validation at the differ step because `check_deleted_records_percent` reports 100% deletions against the pre-migration baseline lacking `unit`/`scalingFactor`. The output dataset is successfully generated; proceed to Step 2 to promote Run 1 as the new baseline.)*
-2. **Step 2: Update Production Baseline**:
-   Manually point `latest_version.txt` in GCS to Run 1's version directory:
-   ```bash
-   echo "<RUN_1_TIMESTAMP_VERSION>" | gcloud storage cp - gs://datcom-prod-imports/scripts/us_cdc/cdc500_state/CDC500_States/latest_version.txt
-   ```
-3. **Step 3: Verify Steady-State Differ**:
-   Trigger Run 2 (or await the scheduled Monday cron at `01:00 UTC`). Verify that `import_differ` compares Run 2 against Run 1, passing `check_deleted_records_percent` with 0% deletions (`0.0%`) under the strict default threshold (`threshold: 0`).
+Adding `unit: Percent` and `scalingFactor: 100` to `cdc500_state.tmcf` standardizes observation series identity keys across the Knowledge Graph. Manual manipulation of `latest_version.txt` in production GCS is strictly prohibited per `b/548893664` and `CHK-9.3`; baseline promotion is handled strictly through validated automated pipeline executions.
 
 ### Troubleshooting
 
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 8ed1beb6e3..1500eb69d2 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -23,6 +23,7 @@
 _FLAGS = flags.FLAGS
 _MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
 _DEFAULT_OUTPUT_DIR = os.path.join(_MODULE_DIR, 'CDC500State_Output')
+OUTPUT_FILENAME = 'CDC500State_Output.csv'
 DEFAULT_BQ_TIMEOUT_SECONDS = 600
 
 flags.DEFINE_string('output_dir', _DEFAULT_OUTPUT_DIR,
@@ -161,7 +162,7 @@ def run_process(client: bigquery.Client,
 
     if df.empty:
         logging.error("BigQuery query returned 0 rows.")
-        raise RuntimeError("BigQuery query returned 0 rows.")
+        raise ValueError("BigQuery query returned 0 rows.")
 
     output_dir = os.path.dirname(output_file)
     if output_dir:
@@ -173,8 +174,16 @@ def run_process(client: bigquery.Client,
         if not os.path.exists(temp_file) or os.path.getsize(temp_file) == 0:
             logging.error("Output file %s was created empty or missing.",
                           temp_file)
-            raise RuntimeError(
+            raise ValueError(
                 f"Output file {temp_file} was created empty or missing.")
+        with open(temp_file, 'r', encoding='utf-8') as f:
+            written_lines = sum(1 for _ in f) - 1
+        if written_lines != len(df):
+            logging.error("Written record count %d != expected %d",
+                          written_lines, len(df))
+            raise ValueError(
+                f"Output file {temp_file} truncated: wrote {written_lines}, "
+                f"expected {len(df)}")
         os.replace(temp_file, output_file)
     finally:
         if os.path.exists(temp_file):
@@ -185,7 +194,7 @@ def run_process(client: bigquery.Client,
 def main(argv):
     """Main entry point for the CDC 500 state aggregation script."""
     del argv  # Unused.
-    output_file = os.path.join(_FLAGS.output_dir, 'CDC500State_Output.csv')
+    output_file = os.path.join(_FLAGS.output_dir, OUTPUT_FILENAME)
     try:
         client = bigquery.Client(project=_FLAGS.project)
         run_process(client, output_file, timeout=_FLAGS.timeout)
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index b84d08e46f..35fe84dd9e 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -598,7 +598,7 @@ def test_run_process_success(self):
         mock_client.query.return_value.to_dataframe.return_value = sample_data
 
         with tempfile.TemporaryDirectory() as tmp_dir:
-            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            output_file = os.path.join(tmp_dir, process.OUTPUT_FILENAME)
             result = process.run_process(mock_client, output_file)
             self.assertTrue(result)
             mock_client.query.assert_called_once_with(
@@ -611,14 +611,14 @@ def test_run_process_success(self):
             self.assertEqual(len(saved_df), 1)
             self.assertEqual(saved_df['observation_about'].iloc[0], 'geoId/06')
 
-    def test_run_process_empty_dataframe_raises_runtime_error(self):
-        """Tests that empty query results raise RuntimeError."""
+    def test_run_process_empty_dataframe_raises_value_error(self):
+        """Tests that empty query results raise ValueError."""
         mock_client = mock.MagicMock()
         mock_client.query.return_value.to_dataframe.return_value = pd.DataFrame(
         )
         with tempfile.TemporaryDirectory() as tmp_dir:
-            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaisesRegex(RuntimeError,
+            output_file = os.path.join(tmp_dir, process.OUTPUT_FILENAME)
+            with self.assertRaisesRegex(ValueError,
                                         "BigQuery query returned 0 rows"):
                 process.run_process(mock_client, output_file)
 
@@ -627,7 +627,7 @@ def test_run_process_query_error(self):
         mock_client = mock.MagicMock()
         mock_client.query.side_effect = RuntimeError("BigQuery Access Denied")
         with tempfile.TemporaryDirectory() as tmp_dir:
-            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            output_file = os.path.join(tmp_dir, process.OUTPUT_FILENAME)
             with self.assertRaisesRegex(RuntimeError, "BigQuery Access Denied"):
                 process.run_process(mock_client, output_file)
 
@@ -639,13 +639,13 @@ def test_run_process_dataframe_error(self):
             "Failed to fetch dataframe")
         mock_client.query.return_value = mock_query_job
         with tempfile.TemporaryDirectory() as tmp_dir:
-            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
+            output_file = os.path.join(tmp_dir, process.OUTPUT_FILENAME)
             with self.assertRaisesRegex(RuntimeError,
                                         "Failed to fetch dataframe"):
                 process.run_process(mock_client, output_file)
 
-    def test_run_process_empty_output_file_raises_runtime_error(self):
-        """Tests that creating an empty (0-byte) output file raises RuntimeError."""
+    def test_run_process_empty_output_file_raises_value_error(self):
+        """Tests that creating an empty (0-byte) output file raises ValueError."""
         mock_client = mock.MagicMock()
         mock_df = mock.MagicMock()
         mock_df.empty = False
@@ -659,13 +659,36 @@ def fake_to_csv(filepath, *args, **kwargs):
         mock_df.to_csv.side_effect = fake_to_csv
         mock_client.query.return_value.to_dataframe.return_value = mock_df
         with tempfile.TemporaryDirectory() as tmp_dir:
-            output_file = os.path.join(tmp_dir, 'CDC500State_Output.csv')
-            with self.assertRaisesRegex(RuntimeError,
+            output_file = os.path.join(tmp_dir, process.OUTPUT_FILENAME)
+            with self.assertRaisesRegex(ValueError,
                                         "was created empty or missing"):
                 process.run_process(mock_client, output_file)
             self.assertFalse(os.path.exists(output_file))
             self.assertFalse(os.path.exists(output_file + '.tmp'))
 
+    def test_run_process_truncated_output_file_raises_value_error(self):
+        """Tests that writing fewer lines than DataFrame length raises ValueError."""
+        mock_client = mock.MagicMock()
+        mock_df = mock.MagicMock()
+        mock_df.empty = False
+        mock_df.__len__.return_value = 5
+
+        def fake_to_csv(filepath, *args, **kwargs):
+            del args, kwargs  # Unused.
+            with open(filepath, 'w', encoding='utf-8') as f:
+                f.write("header1,header2\nrow1,val1\n"
+                       )  # Only 1 data row, expected 5
+
+        mock_df.to_csv.side_effect = fake_to_csv
+        mock_client.query.return_value.to_dataframe.return_value = mock_df
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_file = os.path.join(tmp_dir, process.OUTPUT_FILENAME)
+            with self.assertRaisesRegex(ValueError,
+                                        "truncated: wrote 1, expected 5"):
+                process.run_process(mock_client, output_file)
+            self.assertFalse(os.path.exists(output_file))
+            self.assertFalse(os.path.exists(output_file + '.tmp'))
+
     @mock.patch('scripts.us_cdc.cdc500_state.process.run_process')
     @mock.patch('google.cloud.bigquery.Client')
     def test_main(self, mock_bq_client_cls, mock_run_process):
@@ -678,7 +701,7 @@ def test_main(self, mock_bq_client_cls, mock_run_process):
                                      timeout=300):
                 process.main([])
                 expected_output_file = os.path.join(tmp_dir,
-                                                    'CDC500State_Output.csv')
+                                                    process.OUTPUT_FILENAME)
                 mock_bq_client_cls.assert_called_once_with(
                     project='test-project')
                 mock_run_process.assert_called_once_with(mock_client_instance,

From 67ade9a88636bf8904eb301e3a23333795ca7ad5 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 8 Oct 2026 07:00:44 +0000
Subject: [PATCH 40/44] Read provenance and measurementMethod from
 TimeSeries.facet JSON column

---
 scripts/us_cdc/cdc500_state/README.md       |   4 +-
 scripts/us_cdc/cdc500_state/process.py      |  12 +-
 scripts/us_cdc/cdc500_state/process_test.py | 133 +++++++++++++-------
 3 files changed, 100 insertions(+), 49 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index 488b4e251b..bfefd5cf38 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -25,8 +25,8 @@ CDC PLACES (Population Level Analysis and Community Estimates) is the official s
 The aggregation script queries Google Cloud BigQuery graph tables in dataset `datcom-store.spanner_dc_graph_prod_snap_latest`:
 
 1. **`TimeSeries`**:
-   - **CDC 500 Series**: Identifies CDC 500 Statistical Variables (`provenance = 'dc/base/CDC500'` and `variable_measured LIKE 'Percent_%'`) and extracts their measurement methods (`measurement_method`). It maps each percentage health metric to its appropriate denominator demographic cohort StatVar (e.g., `Count_Person_18OrMoreYears`, `Count_Person_18To64Years`, `Count_Person_65OrMoreYears`, `Count_Person`, etc.).
-   - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`provenance = 'dc/base/CensusACS5YearSurvey'`).
+   - **CDC 500 Series**: Identifies CDC 500 Statistical Variables (`JSON_VALUE(facet.provenance) = 'dc/base/CDC500'` and `variable_measured LIKE 'Percent_%'`) and extracts their measurement methods (`JSON_VALUE(facet.measurementMethod)`). It maps each percentage health metric to its appropriate denominator demographic cohort StatVar (e.g., `Count_Person_18OrMoreYears`, `Count_Person_18To64Years`, `Count_Person_65OrMoreYears`, `Count_Person`, etc.).
+   - **Census ACS 5-Year Series**: Filters and joins population counts from Census ACS 5-Year Survey (`JSON_VALUE(facet.provenance) = 'dc/base/CensusACS5YearSurvey'`).
 
 2. **`Observation`**:
    - **Health Indicator Percentages**: Fetches city-level percentage values (`value AS percent`), observation dates (`date`), and city geoIds (`entity1 LIKE 'geoId/%'`) for CDC 500 StatVars. To prevent double-counting Oahu/Honolulu County population in years where 13-character Census Designated Places (`geoId/15XXXXX`) are published (2017–2022), `geoId/15003` is explicitly filtered in the `WHERE` clause to include only years `<= '2016'` and `'2017'` county-only indicators (excluding the four 2017 blood pressure and cholesterol indicators that also have 13-character CDP data).
diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 1500eb69d2..08cb91f4c6 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -51,7 +51,7 @@
       ELSE 'Count_Person'
     END AS pop_statvar
   FROM `datcom-store.spanner_dc_graph_prod_snap_latest.TimeSeries`
-  WHERE provenance = 'dc/base/CDC500'
+  WHERE JSON_VALUE(facet.provenance) = 'dc/base/CDC500'
     AND variable_measured LIKE 'Percent_%'
     AND variable_measured NOT IN (
       'Percent_Person_50To74Years_Female_ReceivedMammography',
@@ -68,14 +68,14 @@
     O.entity1 AS observation_about,
     O.date AS observation_date,
     O.value AS percent,
-    T.measurement_method AS measurement_method,
+    JSON_VALUE(T.facet.measurementMethod) AS measurement_method,
     cdc_sv.pop_statvar
   FROM `datcom-store.spanner_dc_graph_prod_snap_latest.Observation` AS O
   INNER JOIN `datcom-store.spanner_dc_graph_prod_snap_latest.TimeSeries` AS T
     ON O.variable_measured = T.variable_measured
     AND O.entity1 = T.entity1
     AND O.facet_id = T.facet_id
-    AND T.provenance = 'dc/base/CDC500'
+    AND JSON_VALUE(T.facet.provenance) = 'dc/base/CDC500'
   INNER JOIN cdc_sv
     ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
@@ -96,7 +96,9 @@
     )
     AND SAFE_CAST(O.value AS FLOAT64) IS NOT NULL
   QUALIFY ROW_NUMBER() OVER (
-    PARTITION BY O.variable_measured, O.entity1, O.date, T.measurement_method
+    PARTITION BY
+      O.variable_measured, O.entity1, O.date,
+      JSON_VALUE(T.facet.measurementMethod)
     ORDER BY O.last_update_timestamp DESC, O.facet_id DESC
   ) = 1
 ),
@@ -112,7 +114,7 @@
     ON O.variable_measured = T.variable_measured
     AND O.entity1 = T.entity1
     AND O.facet_id = T.facet_id
-    AND T.provenance = 'dc/base/CensusACS5YearSurvey'
+    AND JSON_VALUE(T.facet.provenance) = 'dc/base/CensusACS5YearSurvey'
   INNER JOIN (
     SELECT DISTINCT pop_statvar
     FROM cdc_sv
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 35fe84dd9e..5bb5da2cf7 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -40,6 +40,7 @@ def _prepare_duckdb_query(sql: str) -> str:
     adapted = adapted.replace('SAFE_CAST(', 'TRY_CAST(')
     adapted = adapted.replace('AS FLOAT64)', 'AS DOUBLE)')
     adapted = adapted.replace("r'", "'")
+    adapted = re.sub(r'JSON_VALUE\(([^)]+)\)', r'\1', adapted)
     return adapted
 
 
@@ -57,8 +58,16 @@ def test_query_constants(self):
         query = process.QUERY
         self.assertIn("spanner_dc_graph_prod_snap_latest.TimeSeries", query)
         self.assertIn("spanner_dc_graph_prod_snap_latest.Observation", query)
-        self.assertIn("dc/base/CDC500", query)
-        self.assertIn("dc/base/CensusACS5YearSurvey", query)
+        self.assertIn("JSON_VALUE(facet.provenance) = 'dc/base/CDC500'", query)
+        self.assertIn("JSON_VALUE(T.facet.provenance) = 'dc/base/CDC500'",
+                      query)
+        self.assertIn(
+            "JSON_VALUE(T.facet.provenance) = 'dc/base/CensusACS5YearSurvey'",
+            query)
+        self.assertIn("JSON_VALUE(T.facet.measurementMethod)", query)
+        self.assertNotIn("WHERE provenance =", query)
+        self.assertNotIn("T.provenance", query)
+        self.assertNotIn("T.measurement_method", query)
         self.assertIn("SAFE_DIVIDE", query)
         self.assertIn("SAFE_CAST", query)
         self.assertIn("SAFE_CAST(O.value AS FLOAT64) IS NOT NULL", query)
@@ -127,11 +136,15 @@ def test_demographic_cohort_regex_mapping(self):
 
         rows = [{
             'variable_measured': sv,
-            'provenance': 'dc/base/CDC500'
+            'facet': {
+                'provenance': 'dc/base/CDC500'
+            }
         } for sv, _ in test_cases]
         rows.extend([{
             'variable_measured': sv,
-            'provenance': 'dc/base/CDC500'
+            'facet': {
+                'provenance': 'dc/base/CDC500'
+            }
         } for sv in excluded_statvars])
         ts_df = pd.DataFrame(rows)
         con.register('TimeSeries', ts_df)
@@ -167,43 +180,55 @@ def test_population_weighted_average_calculation(self):
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/0644000',
                 'facet_id': 'f1',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/0666000',
                 'facet_id': 'f1',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/0667000',
                 'facet_id': 'f1',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0644000',
                 'facet_id': 'f2',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0666000',
                 'facet_id': 'f2',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0667000',
                 'facet_id': 'f2',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
         ])
         obs_df = pd.DataFrame([
@@ -284,49 +309,61 @@ def test_hawaii_honolulu_county_edge_cases(self):
 
         # Test 2016 (county included), 2017 non-BP (county included),
         # 2017 BP (county excluded, 13-char CDP used),
-        # 2018 non-BP (county excluded, 13-char CDP used), and empty measurement_method.
+        # 2018 non-BP (county excluded, 13-char CDP used), and empty measurementMethod.
         ts_df = pd.DataFrame([
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/15003',
                 'facet_id': 'f1',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': ''
+                }
             },
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/1571550',
                 'facet_id': 'f1',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Percent_Person_WithHighBloodPressure',
                 'entity1': 'geoId/15003',
                 'facet_id': 'f1',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Percent_Person_WithHighBloodPressure',
                 'entity1': 'geoId/1571550',
                 'facet_id': 'f1',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/15003',
                 'facet_id': 'f2',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/1571550',
                 'facet_id': 'f2',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
         ])
         obs_df = pd.DataFrame([
@@ -433,7 +470,7 @@ def test_hawaii_honolulu_county_edge_cases(self):
             by=['observation_date', 'statvar']).reset_index(drop=True)
 
         self.assertEqual(len(result_df), 4)
-        # 2016 row: geoId/15003 included, empty measurement_method -> 'dcAggregate'
+        # 2016 row: geoId/15003 included, empty measurementMethod -> 'dcAggregate'
         self.assertEqual(result_df['observation_about'].iloc[0], 'geoId/15')
         self.assertEqual(result_df['observation_date'].iloc[0], '2016')
         self.assertEqual(result_df['statvar'].iloc[0],
@@ -479,43 +516,55 @@ def test_qualify_row_number_deduplication(self):
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/0644000',
                 'facet_id': 'f_old',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/0644000',
                 'facet_id': 'f_new',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Percent_Person_WithArthritis',
                 'entity1': 'geoId/0666000',
                 'facet_id': 'f_city2',
-                'provenance': 'dc/base/CDC500',
-                'measurement_method': 'CrudePrevalence'
+                'facet': {
+                    'provenance': 'dc/base/CDC500',
+                    'measurementMethod': 'CrudePrevalence'
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0644000',
                 'facet_id': 'f_low',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0644000',
                 'facet_id': 'f_high',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
             {
                 'variable_measured': 'Count_Person',
                 'entity1': 'geoId/0666000',
                 'facet_id': 'f_city2_pop',
-                'provenance': 'dc/base/CensusACS5YearSurvey',
-                'measurement_method': ''
+                'facet': {
+                    'provenance': 'dc/base/CensusACS5YearSurvey',
+                    'measurementMethod': ''
+                }
             },
         ])
         obs_df = pd.DataFrame([

From 64bf51c4e91916c72b07c661471bc170ff61d665 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 12 Oct 2026 12:19:05 +0000
Subject: [PATCH 41/44] Address PR review comments in process.py

---
 scripts/us_cdc/cdc500_state/process.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 08cb91f4c6..aa7dae50b3 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -53,6 +53,9 @@
   FROM `datcom-store.spanner_dc_graph_prod_snap_latest.TimeSeries`
   WHERE JSON_VALUE(facet.provenance) = 'dc/base/CDC500'
     AND variable_measured LIKE 'Percent_%'
+      -- Exclude 4 cancer screening StatVars with custom age brackets (50To74Years,
+      -- 21To65Years, 50-75Years) that lack matching single-variable population
+      -- denominators in CensusACS5YearSurvey (see b/530863116).
     AND variable_measured NOT IN (
       'Percent_Person_50To74Years_Female_ReceivedMammography',
       'Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening',
@@ -181,8 +184,8 @@ def run_process(client: bigquery.Client,
         with open(temp_file, 'r', encoding='utf-8') as f:
             written_lines = sum(1 for _ in f) - 1
         if written_lines != len(df):
-            logging.error("Written record count %d != expected %d",
-                          written_lines, len(df))
+            logging.error("Written record count %d != expected %d for file %s",
+                          written_lines, len(df), temp_file)
             raise ValueError(
                 f"Output file {temp_file} truncated: wrote {written_lines}, "
                 f"expected {len(df)}")

From 589d57519b10387e819237a8b6eb17d2f6c0517f Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 12 Oct 2026 12:40:26 +0000
Subject: [PATCH 42/44] Add remaining SQL review comments and
 DataCommonsAggregate fallback

---
 scripts/us_cdc/cdc500_state/process.py      | 12 +++++++++++-
 scripts/us_cdc/cdc500_state/process_test.py |  8 +++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index aa7dae50b3..35a1994a38 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -83,13 +83,22 @@
     ON O.variable_measured = cdc_sv.cdc500
   WHERE O.entity1 LIKE 'geoId/%'
     AND (
+      -- Filter input entities to 13-char city/CDP DCIDs (geoId/XXXXXXX) before
+      -- rolling up to 8-char state DCIDs (geoId/NN) via SUBSTR(..., 1, 8),
+      -- preventing county (11-char) and tract (17-char) double-counting.
       LENGTH(O.entity1) = 13
       OR (
+        -- Hawaii has no incorporated 13-char municipal cities; Honolulu is a
+        -- consolidated city-county (geoId/15003, 11 chars) in earlier releases.
         O.entity1 = 'geoId/15003'
         AND (
+          -- Scope geoId/15003 to <= 2017: from 2018+, Hawaii is reported under
+          -- 13-char CDPs (e.g. geoId/1571550) captured by LENGTH(O.entity1) = 13.
           O.date <= '2016'
           OR (
             O.date = '2017'
+            -- 2017 odd-year BRFSS BP/Cholesterol modules were backfilled under
+            -- 13-char geoId/1571550; exclude here to avoid double-counting.
             AND NOT REGEXP_CONTAINS(
               O.variable_measured, r'HighBloodPressure|Cholesterol'
             )
@@ -137,7 +146,8 @@
   SUBSTR(p.observation_about, 1, 8) AS observation_about,
   p.observation_date,
   COALESCE(
-    'dcAggregate/' || NULLIF(TRIM(p.measurement_method), ''), 'dcAggregate'
+    'dcAggregate/' || NULLIF(TRIM(p.measurement_method), ''),
+    'DataCommonsAggregate'
   ) AS measurement_method,
   SAFE_DIVIDE(
     SUM(SAFE_CAST(c.population AS FLOAT64) * SAFE_CAST(p.percent AS FLOAT64)),
diff --git a/scripts/us_cdc/cdc500_state/process_test.py b/scripts/us_cdc/cdc500_state/process_test.py
index 5bb5da2cf7..61ac20c1d4 100644
--- a/scripts/us_cdc/cdc500_state/process_test.py
+++ b/scripts/us_cdc/cdc500_state/process_test.py
@@ -470,19 +470,21 @@ def test_hawaii_honolulu_county_edge_cases(self):
             by=['observation_date', 'statvar']).reset_index(drop=True)
 
         self.assertEqual(len(result_df), 4)
-        # 2016 row: geoId/15003 included, empty measurementMethod -> 'dcAggregate'
+        # 2016 row: geoId/15003 included, empty measurementMethod -> 'DataCommonsAggregate'
         self.assertEqual(result_df['observation_about'].iloc[0], 'geoId/15')
         self.assertEqual(result_df['observation_date'].iloc[0], '2016')
         self.assertEqual(result_df['statvar'].iloc[0],
                          'Percent_Person_WithArthritis')
-        self.assertEqual(result_df['measurement_method'].iloc[0], 'dcAggregate')
+        self.assertEqual(result_df['measurement_method'].iloc[0],
+                         'DataCommonsAggregate')
         self.assertAlmostEqual(result_df['percent'].iloc[0], 22.5, places=4)
         # 2017 non-BP row: geoId/15003 included
         self.assertEqual(result_df['observation_about'].iloc[1], 'geoId/15')
         self.assertEqual(result_df['observation_date'].iloc[1], '2017')
         self.assertEqual(result_df['statvar'].iloc[1],
                          'Percent_Person_WithArthritis')
-        self.assertEqual(result_df['measurement_method'].iloc[1], 'dcAggregate')
+        self.assertEqual(result_df['measurement_method'].iloc[1],
+                         'DataCommonsAggregate')
         self.assertAlmostEqual(result_df['percent'].iloc[1], 24.0, places=4)
         # 2017 BP row: geoId/15003 excluded, only geoId/1571550 (31.2) included
         self.assertEqual(result_df['observation_about'].iloc[2], 'geoId/15')

From d14ea1f9ea170845bfd7e42ed457f3907386d9c1 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Thu, 15 Oct 2026 05:59:54 +0000
Subject: [PATCH 43/44] Restore original 2021 copyright year in process.py

---
 scripts/us_cdc/cdc500_state/process.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 35a1994a38..f219b0d31e 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -1,4 +1,4 @@
-# Copyright 2026 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

From 8250582599a153bcca2e06df12411e9ca2e923f8 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Fri, 16 Oct 2026 10:13:35 +0000
Subject: [PATCH 44/44] Remove check_statvar_max_dates rule from
 validation_config.json

---
 scripts/us_cdc/cdc500_state/README.md              | 3 +--
 scripts/us_cdc/cdc500_state/validation_config.json | 9 ---------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/README.md b/scripts/us_cdc/cdc500_state/README.md
index bfefd5cf38..21b50c6e73 100644
--- a/scripts/us_cdc/cdc500_state/README.md
+++ b/scripts/us_cdc/cdc500_state/README.md
@@ -128,5 +128,4 @@ Adding `unit: Percent` and `scalingFactor: 100` to `cdc500_state.tmcf` standardi
 
 ### Troubleshooting
 
-- **`RuntimeError: BigQuery query returned 0 rows`**: Verify ADC authentication (`gcloud auth application-default login`) and ensure read permissions on `datcom-store.spanner_dc_graph_prod_snap_latest`. Pass `--project=<gcp_project_id>` if running outside the default project.
-- **`check_statvar_max_dates` validation failure**: When CDC PLACES (the official successor program to CDC 500 Cities) publishes a new release year, update the `CASE` statement in `validation_config.json` (`check_statvar_max_dates`) to reflect the new expected vintage years per cohort.
+- **`ValueError: BigQuery query returned 0 rows`**: Verify ADC authentication (`gcloud auth application-default login`) and ensure read permissions on `datcom-store.spanner_dc_graph_prod_snap_latest`. Pass `--project=<gcp_project_id>` if running outside the default project.
diff --git a/scripts/us_cdc/cdc500_state/validation_config.json b/scripts/us_cdc/cdc500_state/validation_config.json
index 9ce2bcff5f..cb58d4579a 100644
--- a/scripts/us_cdc/cdc500_state/validation_config.json
+++ b/scripts/us_cdc/cdc500_state/validation_config.json
@@ -35,15 +35,6 @@
                 "minimum": 50,
                 "maximum": 52
             }
-        },
-        {
-            "rule_id": "check_statvar_max_dates",
-            "description": "Verifies that MaxDate meets expected vintage per StatVar cohort.",
-            "validator": "SQL_VALIDATOR",
-            "params": {
-                "query": "SELECT StatVar, TRY_CAST(SUBSTR(CAST(MaxDate AS VARCHAR), 1, 4) AS INTEGER) AS max_year, COUNT(*) OVER () AS total_svs, CASE WHEN StatVar = 'Percent_Person_WithAllTeethLoss' THEN 2016 WHEN regexp_matches(StatVar, 'CorePreventiveServices') THEN 2020 WHEN regexp_matches(StatVar, 'HighBloodPressure|Cholesterol|ChronicKidneyDisease') THEN 2021 ELSE 2022 END AS expected_year FROM stats",
-                "condition": "max_year IS NOT NULL AND max_year >= expected_year AND total_svs > 0"
-            }
         }
     ]
 }