From 3d77964283c0b0c46c74e1797e9601f32eae5a70 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Mon, 12 Oct 2026 12:22:42 +0000 Subject: [PATCH 1/4] Clean non-numeric characters in S2409 quantity values during processing --- scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py index 924af6d1f2..04713d92d7 100644 --- a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py +++ b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py @@ -171,7 +171,10 @@ def _process_dataframe(self, df, filename): obs_df = pd.DataFrame(columns=self.csv_columns) obs_df['Place'] = place_geoIds obs_df['StatVar'] = column_map[column]['Node'] - obs_df['Quantity'] = df[column].values.tolist() + # Clean the quantity values by removing commas, dashes, and any non-numeric characters like '+' + obs_df['Quantity'] = df[column].apply(lambda x: str(x).replace( + ',', '').replace('-', '').replace('+', '')).astype( + float).tolist() # add unit to the csv try: unit = column_map[column]['unit'] From eb88324e8330e0a48de8c3331219aa92e3ae8e1f Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Mon, 12 Oct 2026 12:40:00 +0000 Subject: [PATCH 2/4] Address review feedback: use vectorized str.replace and pd.to_numeric with errors='coerce' --- .../us_census/acs5yr/subject_tables/s2409/data_loader.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py index 04713d92d7..36d92dffed 100644 --- a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py +++ b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py @@ -172,9 +172,11 @@ def _process_dataframe(self, df, filename): obs_df['Place'] = place_geoIds obs_df['StatVar'] = column_map[column]['Node'] # Clean the quantity values by removing commas, dashes, and any non-numeric characters like '+' - obs_df['Quantity'] = df[column].apply(lambda x: str(x).replace( - ',', '').replace('-', '').replace('+', '')).astype( - float).tolist() + obs_df['Quantity'] = pd.to_numeric( + df[column].astype(str).str.replace(r'[,+-]', + '', + regex=True), + errors='coerce').tolist() # add unit to the csv try: unit = column_map[column]['unit'] From 2bb8c03b727c1910a07490f49abf9940b1d302a4 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Thu, 15 Oct 2026 12:50:14 +0000 Subject: [PATCH 3/4] Fix yapf formatting in S2409 data_loader.py --- scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py index 36d92dffed..9d4d2d0cbe 100644 --- a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py +++ b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py @@ -173,8 +173,7 @@ def _process_dataframe(self, df, filename): obs_df['StatVar'] = column_map[column]['Node'] # Clean the quantity values by removing commas, dashes, and any non-numeric characters like '+' obs_df['Quantity'] = pd.to_numeric( - df[column].astype(str).str.replace(r'[,+-]', - '', + df[column].astype(str).str.replace(r'[,+-]', '', regex=True), errors='coerce').tolist() # add unit to the csv From 218d0235b1311bb37f1a5e984f849d2a8c30cc80 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Mon, 19 Oct 2026 05:45:26 +0000 Subject: [PATCH 4/4] Refine S2409 quantity cleaning regex, add unit test coverage, and update README --- .../acs5yr/subject_tables/s2409/README.md | 3 +- .../subject_tables/s2409/data_loader.py | 5 +- .../s2409/subject_table_s2409_test.py | 46 +++++++++++++++++-- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/scripts/us_census/acs5yr/subject_tables/s2409/README.md b/scripts/us_census/acs5yr/subject_tables/s2409/README.md index 265388606a..ae9b772198 100644 --- a/scripts/us_census/acs5yr/subject_tables/s2409/README.md +++ b/scripts/us_census/acs5yr/subject_tables/s2409/README.md @@ -3,10 +3,11 @@ This import includes the class of worker characteristics for full-time, year-round, 16 year and over civilian employed population characterized by the gender. For years 2010-2014, Median Earnings (in Dollars Inflation Adjusted) are also available. -Years: 2010-2019 +Years: 2010-2024 Geo : Country, State, County and Place Important Notes : 1. Counts of Males and Females were available as percentages for years 2010-2014. Same are converted to absolute values using 'denominators' section in json spec. 2. Local import scripts s2409/process.py and s2409/data_loader.py are created to process denominators section of json spec only for years 2010-2014. 3. Margin of Error for Median Earnings is not processed in this import. +4. Top-coded (`250,000+`) and bottom-coded (`2,500-`) Median Earnings values and comma separators are cleaned by stripping commas and trailing `+`/`-` markers and coercing to numeric (`float`) values during processing. diff --git a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py index 9d4d2d0cbe..fc533edba5 100644 --- a/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py +++ b/scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py @@ -171,9 +171,10 @@ def _process_dataframe(self, df, filename): obs_df = pd.DataFrame(columns=self.csv_columns) obs_df['Place'] = place_geoIds obs_df['StatVar'] = column_map[column]['Node'] - # Clean the quantity values by removing commas, dashes, and any non-numeric characters like '+' + # Clean the quantity values by removing commas and trailing top/bottom-coding markers ('+' or '-') obs_df['Quantity'] = pd.to_numeric( - df[column].astype(str).str.replace(r'[,+-]', '', + df[column].astype(str).str.replace(r',|[+-]$', + '', regex=True), errors='coerce').tolist() # add unit to the csv diff --git a/scripts/us_census/acs5yr/subject_tables/s2409/subject_table_s2409_test.py b/scripts/us_census/acs5yr/subject_tables/s2409/subject_table_s2409_test.py index 2af5d22f8f..8a78c8571a 100644 --- a/scripts/us_census/acs5yr/subject_tables/s2409/subject_table_s2409_test.py +++ b/scripts/us_census/acs5yr/subject_tables/s2409/subject_table_s2409_test.py @@ -17,14 +17,15 @@ import sys import tempfile import unittest - -from .process import set_column_map -from .data_loader import process_subject_tables +import pandas as pd # Allows the unittest to access table directories in relative path. Also used to # import modules for generating cleaned CSV, MCF and Column Map _SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) -# sys.path.append(os.path.join(_SCRIPT_PATH, '..')) +sys.path.append(os.path.join(_SCRIPT_PATH, '..')) + +from s2409.process import set_column_map +from s2409.data_loader import process_subject_tables table_dir_path = os.path.join(_SCRIPT_PATH, '.') @@ -121,3 +122,40 @@ def test_csv_mcf_column_map(self): test_csv, expected_csv = _read_files(test_csv_path, paths['csv']) self.assertEqual(test_csv, expected_csv) + + def test_quantity_cleaning(self): + paths = _get_paths(table_dir_path) + col_name = ('Median earnings (dollars)!!Estimate!!Full-time, year-round' + ' civilian employed population 16 years and over') + + with tempfile.TemporaryDirectory() as tmp_dir: + csv_input_path = os.path.join( + tmp_dir, 'ACSST5Y2010.S2409_data_with_overlays_test.csv') + rows = [ + ['GEO_ID', 'NAME', 'S2409_C04_001E'], + ['id', 'Geographic Area Name', col_name], + ['0400000US01', 'Alabama', '2,500-'], + ['0400000US02', 'Alaska', '250,000+'], + ['0400000US04', 'Arizona', '45,678'], + ['0400000US05', 'Arkansas', '1.25e-02'], + ['0400000US06', 'California', '-'], + ['0400000US08', 'Colorado', 'N'], + ['0400000US09', 'Connecticut', 'unparseable_val'], + ] + pd.DataFrame(rows).to_csv(csv_input_path, header=False, index=False) + + process_subject_tables(table_prefix='clean_test', + input_path=csv_input_path, + output_dir=tmp_dir, + column_map_path=paths['cmap'], + spec_path=paths['spec'], + debug=False, + delimiter='!!', + has_percent=False) + + cleaned_df = pd.read_csv( + os.path.join(tmp_dir, 'clean_test_cleaned.csv')) + self.assertEqual(cleaned_df['Place'].tolist(), + ['geoId/01', 'geoId/02', 'geoId/04', 'geoId/05']) + self.assertEqual(cleaned_df['Quantity'].tolist(), + [2500.0, 250000.0, 45678.0, 0.0125])