US Census ACS5yr S2409: Clean non-numeric characters in Quantity values

scripts/us_census/acs5yr/subject_tables/s2409/README.md

@@ -3,10 +3,11 @@
 This import includes the class of worker characteristics for full-time, year-round, 16 year and over civilian employed population characterized by the gender.
 For years 2010-2014, Median Earnings (in Dollars Inflation Adjusted) are also available.
 
-Years: 2010-2019
+Years: 2010-2024
 Geo : Country, State, County and Place
 
 Important Notes :
 1. Counts of Males and Females were available as percentages for years 2010-2014. Same are converted to absolute values using 'denominators' section in json spec.
 2. Local import scripts s2409/process.py and s2409/data_loader.py are created to process denominators section of json spec only for years 2010-2014.
 3. Margin of Error for Median Earnings is not processed in this import.
+4. Top-coded (`250,000+`) and bottom-coded (`2,500-`) Median Earnings values and comma separators are cleaned by stripping commas and trailing `+`/`-` markers and coercing to numeric (`float`) values during processing.

scripts/us_census/acs5yr/subject_tables/s2409/data_loader.py

@@ -171,7 +171,12 @@ def _process_dataframe(self, df, filename):
                 obs_df = pd.DataFrame(columns=self.csv_columns)
                 obs_df['Place'] = place_geoIds
                 obs_df['StatVar'] = column_map[column]['Node']
-                obs_df['Quantity'] = df[column].values.tolist()
+                # Clean the quantity values by removing commas and trailing top/bottom-coding markers ('+' or '-')
+                obs_df['Quantity'] = pd.to_numeric(
+                    df[column].astype(str).str.replace(r',|[+-]$',
+                                                       '',
+                                                       regex=True),
+                    errors='coerce').tolist()
                 # add unit to the csv
                 try:
                     unit = column_map[column]['unit']

scripts/us_census/acs5yr/subject_tables/s2409/subject_table_s2409_test.py

@@ -17,14 +17,15 @@
 import sys
 import tempfile
 import unittest
-
-from .process import set_column_map
-from .data_loader import process_subject_tables
+import pandas as pd
 
 # Allows the unittest to access table directories in relative path. Also used to
 # import modules for generating cleaned CSV, MCF and Column Map
 _SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
-# sys.path.append(os.path.join(_SCRIPT_PATH, '..'))
+sys.path.append(os.path.join(_SCRIPT_PATH, '..'))
+
+from s2409.process import set_column_map
+from s2409.data_loader import process_subject_tables
 
 table_dir_path = os.path.join(_SCRIPT_PATH, '.')
 
@@ -121,3 +122,40 @@ def test_csv_mcf_column_map(self):
                 test_csv, expected_csv = _read_files(test_csv_path,
                                                      paths['csv'])
                 self.assertEqual(test_csv, expected_csv)
+
+    def test_quantity_cleaning(self):
+        paths = _get_paths(table_dir_path)
+        col_name = ('Median earnings (dollars)!!Estimate!!Full-time, year-round'
+                    ' civilian employed population 16 years and over')
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            csv_input_path = os.path.join(
+                tmp_dir, 'ACSST5Y2010.S2409_data_with_overlays_test.csv')
+            rows = [
+                ['GEO_ID', 'NAME', 'S2409_C04_001E'],
+                ['id', 'Geographic Area Name', col_name],
+                ['0400000US01', 'Alabama', '2,500-'],
+                ['0400000US02', 'Alaska', '250,000+'],
+                ['0400000US04', 'Arizona', '45,678'],
+                ['0400000US05', 'Arkansas', '1.25e-02'],
+                ['0400000US06', 'California', '-'],
+                ['0400000US08', 'Colorado', 'N'],
+                ['0400000US09', 'Connecticut', 'unparseable_val'],
+            ]
+            pd.DataFrame(rows).to_csv(csv_input_path, header=False, index=False)
+
+            process_subject_tables(table_prefix='clean_test',
+                                   input_path=csv_input_path,
+                                   output_dir=tmp_dir,
+                                   column_map_path=paths['cmap'],
+                                   spec_path=paths['spec'],
+                                   debug=False,
+                                   delimiter='!!',
+                                   has_percent=False)
+
+            cleaned_df = pd.read_csv(
+                os.path.join(tmp_dir, 'clean_test_cleaned.csv'))
+            self.assertEqual(cleaned_df['Place'].tolist(),
+                             ['geoId/01', 'geoId/02', 'geoId/04', 'geoId/05'])
+            self.assertEqual(cleaned_df['Quantity'].tolist(),
+                             [2500.0, 250000.0, 45678.0, 0.0125])