From 5743b57fd6913b40f815252d254794f8fe489a1f Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 4 Sep 2026 13:27:32 +0000 Subject: [PATCH 01/20] Modified download script and handle deletions --- .../environment_pvmap.csv | 2 + .../india_rbi_state_statistics/manifest.json | 57 ++++++++++----- .../rbi_download.py | 50 +++++++------ .../rbi_download_test.py | 71 +++++++++++++++++++ .../rbi_places_resolver.csv | 3 +- .../validation_config.json | 13 ++++ 6 files changed, 155 insertions(+), 41 deletions(-) create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 0a1f8a72a5..4ea5beff8a 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -6,6 +6,8 @@ Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProper Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, +"Haryana, Delhi & Chandigarh",observationAbout,dcid:wikidataId/Q1174 +"Haryana, Delhi and Chandigarh",observationAbout,dcid:wikidataId/Q1174 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index efeca24ba2..0542cc79e7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -20,78 +20,97 @@ "import_inputs": [ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", - "cleaned_csv": "output_files/agriculture/agriculture_output.csv" + "cleaned_csv": "output_files/agriculture/agriculture_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv" + "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", - "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv" + "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", + "node_mcf": "output_files/*/*.mcf" } ], - "cron_schedule": "0 10 * * 1" + "cron_schedule": "0 10 * * 1", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index e30edc2db1..b6c7c654aa 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import tempfile from absl import app, logging, flags import sys from google.cloud import storage @@ -80,26 +81,33 @@ def preprocess_files(directory_path): logging.info(f"Processing file: {file_path}") try: all_sheets_data = pd.read_excel(file_path, sheet_name=None, header=None) - with pd.ExcelWriter(file_path, engine='openpyxl') as writer: - for sheet_name, df in all_sheets_data.items(): - df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip()) - mask = df.eq('State/Union Territory').any(axis=1) - state_positions = df[mask] == 'State/Union Territory' - - def safe_to_numeric(val): - val_str = str(val) - if val_str.isdigit(): - return int(val_str) - try: - float_val = float(val_str) - return float_val - except ValueError: - return val - - df_num = df[mask].applymap(safe_to_numeric) - converted = df_num.mask(state_positions, 'State/Union Territory') - df.loc[mask, :] = converted - df.to_excel(writer, sheet_name=sheet_name, index=False, header=False) + def safe_to_numeric(val): + val_str = str(val) + if val_str.isdigit(): + return int(val_str) + try: + float_val = float(val_str) + return float_val + except ValueError: + return val + + for sheet_name, df in all_sheets_data.items(): + df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip()) + mask = df.eq('State/Union Territory').any(axis=1) + state_positions = df[mask] == 'State/Union Territory' + + df_num = df[mask].map(safe_to_numeric) + converted = df_num.mask(state_positions, 'State/Union Territory') + df = df.astype(object) + df.loc[mask, :] = converted + all_sheets_data[sheet_name] = df + + with tempfile.TemporaryDirectory(dir=directory_path) as output_dir: + output_path = os.path.join(output_dir, file_name) + with pd.ExcelWriter(output_path, engine='openpyxl') as writer: + for sheet_name, df in all_sheets_data.items(): + df.to_excel(writer, sheet_name=sheet_name, index=False, header=False) + os.replace(output_path, file_path) except Exception as e: logging.fatal(f"Error processing {file_name}: {e}") @@ -119,4 +127,4 @@ def main(_): if __name__ == "__main__": - app.run(main) + app.run(main) \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py new file mode 100644 index 0000000000..d7894865cd --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -0,0 +1,71 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import tempfile +from unittest import mock + +from absl.testing import absltest +import pandas as pd + +from statvar_imports.database_on_indian_economy.india_rbi_state_statistics import rbi_download + + +class PreprocessFilesTest(absltest.TestCase): + + def test_preserves_workbook_and_original_error(self): + with tempfile.TemporaryDirectory() as directory: + workbook = pathlib.Path(directory) / 'source.xlsx' + workbook.write_bytes(b'original workbook') + sheets = {'Sheet1': pd.DataFrame([['State/Union Territory']])} + + with mock.patch.object( + rbi_download.pd, 'read_excel', return_value=sheets), \ + mock.patch.object( + pd.DataFrame, + 'map', + side_effect=ValueError('transform failed')), \ + mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files(directory) + + self.assertEqual(workbook.read_bytes(), b'original workbook') + fatal.assert_called_once_with( + 'Error processing source.xlsx: transform failed') + + def test_preprocess_files_converts_numeric_headers(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2015*', '2016@', '2017-18'], + ['Andhra Pradesh', '10.5', '20.0', '30.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertEqual(processed.iloc[0, 0], 'State/Union Territory') + self.assertEqual(processed.iloc[0, 1], 2015) + self.assertEqual(processed.iloc[0, 2], 2016) + self.assertEqual(processed.iloc[0, 3], '2017-18') + self.assertEqual(processed.iloc[1, 0], 'Andhra Pradesh') + + +if __name__ == '__main__': + absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index a67a4fa4f1..2b8594585f 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -8,7 +8,7 @@ Chandigarh,wikidataId/Q43433 Chhattisgarh,wikidataId/Q1168 Dadra & Nagar Haveli,wikidataId/Q46107 Daman & Diu,wikidataId/Q66710 -NCT of Delhi,wikidataId/Q1352 +NCT of Delhi,wikidataId/Q1353 Delhi,wikidataId/Q1353 Goa,wikidataId/Q1171 Gujarat,wikidataId/Q1061 @@ -38,6 +38,7 @@ Uttarakhand,wikidataId/Q1499 West Bengal,wikidataId/Q1356 ALL INDIA,country/IND All-India,country/IND +All India,country/IND Ladakh,wikidataId/Q200667 Kolkata**,wikidataId/Q1348 Chennai**,wikidataId/Q1352 diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json new file mode 100644 index 0000000000..c391694e7c --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 7 + } + } + ] +} From 90610e8e3a325eb86adf2890c3954ca1dbe6f473 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 7 Sep 2026 10:15:31 +0000 Subject: [PATCH 02/20] Made changes suggested by Gemini-bot --- .../india_rbi_state_statistics/manifest.json | 36 +++++++++---------- .../rbi_download.py | 2 +- .../rbi_download_test.py | 20 +++++++++++ .../rbi_places_resolver.csv | 2 +- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 0542cc79e7..af00739432 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -21,92 +21,92 @@ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", "cleaned_csv": "output_files/agriculture/agriculture_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/agriculture/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/price_and_wages/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" } ], "cron_schedule": "0 10 * * 1", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index b6c7c654aa..4c946586d0 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -92,7 +92,7 @@ def safe_to_numeric(val): return val for sheet_name, df in all_sheets_data.items(): - df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip()) + df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip() if pd.notna(x) else x) mask = df.eq('State/Union Territory').any(axis=1) state_positions = df[mask] == 'State/Union Territory' diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index d7894865cd..e2405edbb9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -66,6 +66,26 @@ def test_preprocess_files_converts_numeric_headers(self): self.assertEqual(processed.iloc[0, 3], '2017-18') self.assertEqual(processed.iloc[1, 0], 'Andhra Pradesh') + def test_preprocess_files_preserves_nan(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2015*', None], + ['Andhra Pradesh', None, '30.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertTrue(pd.isna(processed.iloc[0, 2])) + self.assertTrue(pd.isna(processed.iloc[1, 1])) + if __name__ == '__main__': absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index 2b8594585f..7f3c28c3bd 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -8,7 +8,7 @@ Chandigarh,wikidataId/Q43433 Chhattisgarh,wikidataId/Q1168 Dadra & Nagar Haveli,wikidataId/Q46107 Daman & Diu,wikidataId/Q66710 -NCT of Delhi,wikidataId/Q1353 +NCT of Delhi,wikidataId/Q9357528 Delhi,wikidataId/Q1353 Goa,wikidataId/Q1171 Gujarat,wikidataId/Q1061 From 0ede682365749bd597d66efeb9def0fc75e3bb85 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 9 Sep 2026 11:53:28 +0000 Subject: [PATCH 03/20] Changes as per recommendation --- .../india_rbi_state_statistics/README.md | 30 +- .../india_rbi_state_statistics/configs.json | 334 ++++++++++++++++++ .../environment_pvmap.csv | 3 +- .../india_rbi_state_statistics/manifest.json | 3 +- .../rbi_download.py | 309 +++++++++++++--- .../rbi_download_test.py | 219 +++++++++++- .../rbi_places_resolver.csv | 4 +- .../india_rbi_state_statistics/run.sh | 47 ++- .../validation_config.json | 11 +- 9 files changed, 866 insertions(+), 94 deletions(-) create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 19d09757af..96a71547f9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -10,19 +10,22 @@ This import pipeline processes various socio-economic and agricultural statistic ## Configuration -The `rbi_download.py` script relies on a configuration file named `configs.py` to fetch the URLs, filenames, and categories for the data to be downloaded. +The `rbi_download.py` script relies on a local, version-controlled JSON configuration file named `configs.json` to specify URLs, filenames, and categories for the data to be downloaded. -Sample data of the config file: -URLS_CONFIG=[ +Sample structure of `configs.json`: +```json +{ + "URLS_CONFIG": [ { - "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_xxxxxxxxxxxxx.XLSX", - "category": "agriculture", - "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" - }, - ... - ] + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_xxxxxxxxxxxxx.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" + } + ] +} +``` -This approach makes the import process semi-automatic: if the download URLs change in future RBI releases (which commonly happens due to shifting table numbers or publication structures), only this configuration file needs to be updated, rather than modifying the Python script itself. +This makes the import process **Semi-Automated**: if download URLs change in future RBI releases (due to updated publication links or revised tables), only `configs.json` needs to be updated without modifying script logic. Additionally, `rbi_download.py` employs connection pooling with `requests.Session()` + `HTTPAdapter(Retry(...))`, validates file magic bytes (`PK\x03\x04`), and atomically writes downloaded files to prevent corrupted/partial files. ## Data Acquisition and Initial Preprocessing @@ -288,3 +291,10 @@ python3 stat_var_processor.py \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output ``` + +## Validation Configuration and Thresholds + +This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). +- **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. +- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/19k3zyNk_KR9D4dVpXdGvlXKk6AMOzoYU2I1khJSsX8w/edit?resourcekey=0-JR3lDK9eNfo3ftEV6YpGxQ&tab=t.0#heading=h.iiqodx2yoqc8). + diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json new file mode 100644 index 0000000000..52ef784515 --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json @@ -0,0 +1,334 @@ +{ + "URLS_CONFIG": [ + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/53T_111220253720381F6FEB46769E9A3BA1775126F7.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/54T_111220257E02BB1F59FC46F2B936CC773BBC0874.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_net_sown_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/55T_11122025AFBC97CC9BA44E54BC31B3E58DBF2E21.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_irrigated_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/56T_1112202564A24EAD0AA44A278DB069916CA8FECB.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_net_irrigated_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/57T_111220253F1BCFDCDF5047B495821F992E4D87B6.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_cropping_intensity.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_111220250E85BAD71CD64324A6F4E28DD0A05129.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/59T_1112202571AAC5A686F74224A1DF55D2B9D62D77.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/60T_111220251CE8EB785C6E40FAA8BA4EE805D01B93.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/61T_111220251C3BA74BCC3742A7BA09518C4DDD0BE5.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/62T_11122025F9D0951B74274E99A9C236A5CFAC1858.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/63T_1112202514B06350BA2849F98C3551060541C568.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/64T_111220251AE568D097A740F18A52EFEB30BDDA02.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/65T_11122025FF1A1A01BD604A1A9A63E572FF11B5B5.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/66T_1112202551DDDB3EE6624BE9B30EB1692CC0CBA6.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/67T_1112202594892EEE85FF4BAC9AAB5DC35661E62A.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/68T_11122025D202C9C812754B4FBC0F25A348940AA8.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/69T_111220258B2ACEEE047E482AA3EDF0C09F7E3A86.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/70T_11122025A43EF0C6E0154357B8E404D9CF42998C.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/71T_111220252D54659F49874E689923BF46C29E01A5.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/72T_11122025D573A69D2317423FBCFB65F5622AF929.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_fruits.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/73T_11122025E35C4283EAE7430A88B1BF32B236A976.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_vegetables.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/74T_11122025EBF01225C48441C2A23B436CCC589BF1.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_fruits.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/75T_11122025C8A307B91418448DBF5B5E54997E7355.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_vegetables.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/76T_1112202535D900531AEA4C71B96F562B9AE59835.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/77T_11122025DA3881A5159A4760B53AC35C5BFBA8D9.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/78T_111220252516A0D23AA94A6B809CFB3B1953DC7B.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/79T_1112202567B7934A72A04BF988AAC0656AA9DB66.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/80T_111220251421B93FB22A4B8BAE3E52809636B603.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/81T_11122025F795A58AD4964BFD9C1AF74F0557164E.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/82T_1112202534205116BBCA4010AC1792E31FADC9C9.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/83T_111220255A0477BFBDFD45AF93EDA200B2A5A66F.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/84T_11122025D106F74C0F98465DAF95013CB77F3A20.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/85T_111220250039FAA1932944E5B7627528B147FAFC.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/86T_111220258B6992228C504C2D84CBD88574260E87.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/87T_1112202547D1DA4285204EAFA0BAAA6CA47834C7.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/88T_1112202507582D4179654DC292A6311E3BDB2AB5.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/89T_11122025CC34A78CE2D64C1D880A9B4B6FA1135D.XLSX", + "category": "agriculture", + "filename": "state_wise_per_hectare_consumption_of_fertiliser_n_p_k.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/90T_1112202558892C39CD554E4098A8EB79C63D47D9.XLSX", + "category": "agriculture", + "filename": "state_wise_meat_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/91T_11122025A7A12D69EA1645129226BEAE5783E1A4.XLSX", + "category": "agriculture", + "filename": "state_wise_milk_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/92T_11122025867917E0AC5644ADB035966AAF9D220B.XLSX", + "category": "agriculture", + "filename": "state_wise_egg_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/93T_1112202561B6558711FE4088998165985924266E.XLSX", + "category": "agriculture", + "filename": "state_wise_fish_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/94T_111220254142688D61EB497BA75DAFDDB72F7160.XLSX", + "category": "agriculture", + "filename": "state_wise_wool_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/99T_111220251AFF6E04A39B48E499CE9CDE9E66838B.XLSX", + "category": "environment", + "filename": "state_wise_forest_cover.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/101T_11122025C61D6934B6D74891B15B91679CFE1689.XLSX", + "category": "environment", + "filename": "state_wise_tree_cover.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/102T_11122025C0C87B2909234302A1F4A52A9A42E4DF.XLSX", + "category": "environment", + "filename": "sub_division_wise_annual_rainfall.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/105T_11122025454345A0E067418D9326DD30AE67BF6F.XLSX", + "category": "environment", + "filename": "state_wise_expenditure_on_relief_on_natural_calamities.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/106T_111220251B177E8F85CA412DB39DCE90BCBD724A.XLSX", + "category": "environment", + "filename": "state_wise_sustainable_development_goals_score_SDGs.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/138T_111220252D21BC51E5A842E4B25E954391C10A41.XLSX", + "category": "infrastructure", + "filename": "state_wise_per_capita_availability_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/139T_111220256569FF1C11E248B7A8ED64D31C999776.XLSX", + "category": "infrastructure", + "filename": "state_wise_availability_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/140T_111220254D8DA0B69B444492B6E9BAF30F3395C8.XLSX", + "category": "infrastructure", + "filename": "state_wise_installed_capacity_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/141T_11122025615E98A72EA6475CAACFFD12FFC83057.XLSX", + "category": "infrastructure", + "filename": "state_wise_power_requirement.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/144T_111220250497208D070C44219818788DB5BCE35A.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_national_highways.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/145T_111220259E033C52098E4732A6101C940DEF1222.XLSX", + "category": "infrastructure", + "filename": "state_wise_railway_route.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/146T_11122025B89449F4159B420E8305A9509F27FE82.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_roads.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/147T_111220255C153B31AB2E4835ABEEAC91D7494BFA.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_state_highways.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/148T_11122025D741C3CB2CC24F448DC99D907589DD71.XLSX", + "category": "infrastructure", + "filename": "state_wise_electricity_transmission_distribution_losses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/149T_1112202538D6A609C55B4FEFBBC44BB6C5ED20F7.XLSX", + "category": "infrastructure", + "filename": "state_wise_telephones_per_100_population.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/150T_11122025C30EFE7568EA472C8AB0290288B160C0.XLSX", + "category": "infrastructure", + "filename": "state_wise_road_constructed_under_PMGSY.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/108T_111220251942D16B3BED4D73BE851D15D2329168.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_general.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/109T_111220250E067E49350B48659E5A50C3D357BB46.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_food_and_beverages.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/110T_11122025784C1BDC2482496B9E25DA1BA4B38A3D.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_fuel_and_light.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/111T_111220252F7D9704AB664C35A58770BAC73518DE.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_housing_urban.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/112T_11122025771E55EB688044CE8A85E52A901B5D22.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_construction_workers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/113T_111220256FEBCC253D4A47419892925F897E2537.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_general_agricultural_labourers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/114T_111220259CA566982E7F4BF38B3FDAC0D7587F2E.XLSX", + "category": "price_and_wages", + "filename": "state_wise_Average Daily Wage Rates in Rural_india_men_horticulture_workers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/115T_1112202585CF99F21E654CED8990A734A043B3DA.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_non_agricultural_labourers.xlsx" + } + ] +} \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 4ea5beff8a..ebd44f700d 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -6,8 +6,7 @@ Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProper Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, -"Haryana, Delhi & Chandigarh",observationAbout,dcid:wikidataId/Q1174 -"Haryana, Delhi and Chandigarh",observationAbout,dcid:wikidataId/Q1174 +"Haryana, Delhi & Chandigarh",observationAbout,country/IND 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index af00739432..d7f96314e7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -15,7 +15,8 @@ "input_files/agriculture/*.xlsx", "input_files/environment/*.xlsx", "input_files/infrastructure/*.xlsx", - "input_files/price_and_wages/*.xlsx" + "input_files/price_and_wages/*.xlsx", + "counters/*.csv" ], "import_inputs": [ { diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 4c946586d0..7d6da73694 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -12,105 +12,302 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os -import tempfile -from absl import app, logging, flags +import pathlib +import re import sys +import tempfile +import time +from absl import app, flags, logging from google.cloud import storage import pandas as pd +import requests +from requests.adapters import HTTPAdapter +import urllib3 +from urllib3.util import Retry + +# Suppress unverified HTTPS warnings for RBI endpoints +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(SCRIPT_DIR, '../../../util')) INPUT_DIR = os.path.join(SCRIPT_DIR, "input_files") +DEFAULT_CONFIG_JSON = os.path.join(SCRIPT_DIR, 'configs.json') -from download_util_script import _retry_method +flags.DEFINE_string('config_file_path', DEFAULT_CONFIG_JSON, + 'Config file path (local json or gs:// path)') -flags.DEFINE_string( - 'config_file_path', - 'gs://unresolved_mcf/india_rbi/state_statistics/configs.py', - 'Config file path') +DEFAULT_HEADERS = { + 'User-Agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36'), + 'Accept': + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': + 'en-US,en;q=0.9', +} -def reads_config_file(): - _FLAGS = flags.FLAGS - config_file_path = _FLAGS.config_file_path +XLSX_ZIP_SIGNATURE = b'PK\x03\x04' + + +def create_retry_session( + retries: int = 4, + backoff_factor: float = 2.0, + status_forcelist: tuple = (429, 500, 502, 503, 504), +) -> requests.Session: + """Creates a requests.Session with connection pooling, retries, and browser headers.""" + session = requests.Session() + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + allowed_methods=["HEAD", "GET"], + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry_strategy, + pool_connections=10, + pool_maxsize=10) + session.mount("https://", adapter) + session.mount("http://", adapter) + session.headers.update(DEFAULT_HEADERS) + session.verify = False + return session + + +def _load_local_config(file_path): + if not os.path.exists(file_path): + return None try: - storage_client = storage.Client() - bucket_name = config_file_path.split('/')[2] - bucket = storage_client.bucket(bucket_name) - blob_name = '/'.join(config_file_path.split('/')[3:]) - blob = bucket.blob(blob_name) - file_contents = blob.download_as_text() - local_vars = {} - exec(file_contents, {}, local_vars) - return local_vars + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) except Exception as e: - logging.fatal(f"Cannot extract url and related configs: {e}") + logging.warning(f"Failed to read local config file '{file_path}': {e}") + return None + -def download_files(URL_CONFIG): +def reads_config_file(): + _FLAGS = flags.FLAGS try: - for config in URL_CONFIG: - config_url = config.get("url") - category_name = config.get("category") - file_name = config.get("filename") - file_path = os.path.join(INPUT_DIR, category_name, file_name) - os.makedirs(os.path.join(INPUT_DIR, category_name), exist_ok=True) - file_response = _retry_method(config_url, headers=None, tries=3, delay=5, backoff=2) - if file_response: - with open(file_path, 'wb') as f: - f.write(file_response.content) - logging.info(f"Downloaded the file {file_name} successfully.") - except Exception as e: - logging.fatal(f"Download error: {str(e)}") + config_file_path = _FLAGS.config_file_path + except (flags.UnparsedFlagAccessError, AttributeError): + try: + config_file_path = _FLAGS['config_file_path'].value + except Exception: + config_file_path = DEFAULT_CONFIG_JSON + + # 1. If it's a local file path (or doesn't start with gs://), try loading it + if config_file_path and not config_file_path.startswith('gs://'): + config = _load_local_config(config_file_path) + if config and 'URLS_CONFIG' in config: + logging.info(f"Loaded config from local path: {config_file_path}") + return config + logging.warning( + f"Could not load config from local file '{config_file_path}'") + + # 2. If it's a GCS path, try loading from GCS + if config_file_path and config_file_path.startswith('gs://'): + try: + storage_client = storage.Client() + parts = config_file_path[5:].split('/', 1) + bucket_name = parts[0] + blob_name = parts[1] if len(parts) > 1 else '' + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + file_contents = blob.download_as_text() + if config_file_path.endswith('.json'): + return json.loads(file_contents) + local_vars = {} + exec(file_contents, {}, local_vars) + return local_vars + except Exception as e: + logging.warning( + f"Cannot extract url and related configs from GCS '{config_file_path}': {e}. " + "Falling back to local configuration.") + + # 3. Fall back to local configs.json in SCRIPT_DIR + config = _load_local_config(DEFAULT_CONFIG_JSON) + if config and 'URLS_CONFIG' in config: + logging.info( + f"Loaded fallback configuration from {DEFAULT_CONFIG_JSON}") + return config + + logging.error( + "Cannot extract url and related configs: all sources failed.") + raise RuntimeError( + "Cannot extract url and related configs: all sources failed.") + + +def download_files(URL_CONFIG, session=None, delay=0.5): + if not URL_CONFIG: + logging.warning("No URL configurations provided to download.") + return + + if session is None: + session = create_retry_session() + + for config in URL_CONFIG: + config_url = config.get("url") + category_name = config.get("category") + file_name = config.get("filename") + if not config_url or not category_name or not file_name: + logging.warning(f"Skipping incomplete config: {config}") + continue + + target_dir = os.path.join(INPUT_DIR, category_name) + os.makedirs(target_dir, exist_ok=True) + file_path = os.path.join(target_dir, file_name) + + if os.path.exists(file_path): + if os.path.getsize(file_path) > 0: + with open(file_path, 'rb') as f: + magic = f.read(4) + if magic == XLSX_ZIP_SIGNATURE: + logging.info(f"Skipping existing valid file: {file_name}") + continue + # Remove stale, empty, or corrupt file + try: + os.remove(file_path) + except OSError: + pass + + try: + logging.info(f"Attempting GET request to: {config_url}") + response = session.get(config_url, timeout=45) + + if response.status_code == 404: + logging.warning( + f"Table not found (HTTP 404) at {config_url}, skipping {file_name}" + ) + continue + + response.raise_for_status() + content = response.content + + if not content: + logging.error( + f"Empty response received for {file_name} from {config_url}" + ) + continue + + if not content.startswith(XLSX_ZIP_SIGNATURE): + snippet = content[:50].decode('utf-8', errors='replace') + logging.error( + f"Invalid file format received for {file_name} from {config_url}: " + f"expected XLSX ZIP signature, got: {snippet!r}") + continue + + # Write atomically to a temporary file, then move into place + with tempfile.NamedTemporaryFile('wb', + dir=target_dir, + delete=False) as tmp_file: + tmp_file.write(content) + temp_path = tmp_file.name + + os.replace(temp_path, file_path) + logging.info( + f"Downloaded the file {file_name} successfully ({len(content)} bytes)." + ) + + if delay > 0: + time.sleep(delay) + + except Exception as e: + logging.error( + f"Failed to download table {file_name} from {config_url}: {e}") + + +def _apply_map(df, func): + """Version-agnostic element-wise DataFrame mapping (uses df.map if available, else df.applymap).""" + if hasattr(df, 'map'): + return df.map(func) + return df.applymap(func) def preprocess_files(directory_path): if not os.path.isdir(directory_path): - logging.info(f"Error: Directory not found at '{directory_path}'") + logging.fatal(f"Error: Directory not found at '{directory_path}'") return - + xlsx_files = [f for f in os.listdir(directory_path) if f.endswith('.xlsx')] if not xlsx_files: - logging.info(f"No XLSX files found in the directory: {directory_path}") + logging.fatal( + f"No XLSX files found in the directory: {directory_path}") return - logging.info(f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'.") + logging.info( + f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'." + ) - for file_name in xlsx_files: + for file_name in sorted(xlsx_files): file_path = os.path.join(directory_path, file_name) logging.info(f"Processing file: {file_path}") try: - all_sheets_data = pd.read_excel(file_path, sheet_name=None, header=None) + all_sheets_data = pd.read_excel(file_path, + sheet_name=None, + header=None, + engine='openpyxl') + + def clean_cell(val): + if pd.isna(val): + return val + if isinstance(val, (int, float)): + return val + val_str = str(val).replace('*', '').replace('@', '').strip() + if not val_str or val_str.lower() == 'nan': + return float('nan') + return val_str + def safe_to_numeric(val): - val_str = str(val) + if pd.isna(val): + return val + if isinstance(val, (int, float)): + return val + val_str = str(val).strip() + if not val_str or val_str.lower() == 'nan': + return float('nan') if val_str.isdigit(): return int(val_str) try: float_val = float(val_str) + if float_val.is_integer(): + return int(float_val) return float_val except ValueError: return val + def is_state_header(val): + if pd.isna(val): + return False + return bool( + re.search(r'state\s*/\s*union\s*territory', str(val), + re.IGNORECASE)) + for sheet_name, df in all_sheets_data.items(): - df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip() if pd.notna(x) else x) - mask = df.eq('State/Union Territory').any(axis=1) - state_positions = df[mask] == 'State/Union Territory' - - df_num = df[mask].map(safe_to_numeric) - converted = df_num.mask(state_positions, 'State/Union Territory') - df = df.astype(object) - df.loc[mask, :] = converted + df = _apply_map(df, clean_cell) + is_state_cell = _apply_map(df, is_state_header) + mask = is_state_cell.any(axis=1) + + if mask.any(): + state_positions = is_state_cell.loc[mask] + df_num = _apply_map(df[mask], safe_to_numeric) + converted = df_num.mask(state_positions, df[mask]) + df = df.astype(object) + df.loc[mask, :] = converted all_sheets_data[sheet_name] = df with tempfile.TemporaryDirectory(dir=directory_path) as output_dir: output_path = os.path.join(output_dir, file_name) with pd.ExcelWriter(output_path, engine='openpyxl') as writer: for sheet_name, df in all_sheets_data.items(): - df.to_excel(writer, sheet_name=sheet_name, index=False, header=False) + df.to_excel(writer, + sheet_name=sheet_name, + index=False, + header=False) os.replace(output_path, file_path) except Exception as e: - logging.fatal(f"Error processing {file_name}: {e}") + logging.error(f"Error processing {file_name}: {e}") logging.info("All specified XLSX files have been processed.") @@ -120,11 +317,13 @@ def main(_): RBI_URL = configs['URLS_CONFIG'] download_files(RBI_URL) logging.info("Download process Completed successfully") - directories = ['agriculture', 'environment', 'infrastructure', 'price_and_wages'] + directories = [ + 'agriculture', 'environment', 'infrastructure', 'price_and_wages' + ] for directory in directories: preprocess_files(os.path.join(INPUT_DIR, directory)) logging.info("Pre-process Completed successfully") -if __name__ == "__main__": - app.run(main) \ No newline at end of file +if __name__ == "__main__": + app.run(main) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index e2405edbb9..15cce4876f 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -12,14 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os import pathlib +import sys import tempfile from unittest import mock -from absl.testing import absltest +from absl.testing import absltest, flagsaver import pandas as pd +import requests -from statvar_imports.database_on_indian_economy.india_rbi_state_statistics import rbi_download +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_DIR) + +import rbi_download class PreprocessFilesTest(absltest.TestCase): @@ -36,11 +43,11 @@ def test_preserves_workbook_and_original_error(self): pd.DataFrame, 'map', side_effect=ValueError('transform failed')), \ - mock.patch.object(rbi_download.logging, 'fatal') as fatal: + mock.patch.object(rbi_download.logging, 'error') as mock_error: rbi_download.preprocess_files(directory) self.assertEqual(workbook.read_bytes(), b'original workbook') - fatal.assert_called_once_with( + mock_error.assert_called_once_with( 'Error processing source.xlsx: transform failed') def test_preprocess_files_converts_numeric_headers(self): @@ -86,6 +93,210 @@ def test_preprocess_files_preserves_nan(self): self.assertTrue(pd.isna(processed.iloc[0, 2])) self.assertTrue(pd.isna(processed.iloc[1, 1])) + def test_preprocess_files_fatal_on_missing_dir(self): + with mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files('/non/existent/directory/path') + fatal.assert_called_once() + self.assertIn("Directory not found", fatal.call_args[0][0]) + + def test_preprocess_files_fatal_on_empty_dir(self): + with tempfile.TemporaryDirectory() as directory: + with mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files(directory) + fatal.assert_called_once() + self.assertIn("No XLSX files found", fatal.call_args[0][0]) + + def test_preprocess_files_handles_flexible_state_header(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame([['State / Union Territory', '2018-19'], + ['State/Union Territory', '2019-20'], + ['Karnataka', '120.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertEqual(processed.iloc[0, 0], 'State / Union Territory') + self.assertEqual(processed.iloc[1, 0], 'State/Union Territory') + self.assertEqual(processed.iloc[0, 1], '2018-19') + self.assertEqual(processed.iloc[1, 1], '2019-20') + self.assertEqual(processed.iloc[2, 0], 'Karnataka') + self.assertEqual(processed.iloc[2, 1], '120.5') + + def test_preprocess_files_avoids_literal_nan_and_preserves_numeric(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2020', '2021', '2022'], + ['Bihar', 100, 200.5, ' '], ['Assam', 'nan', 'NAN', '300']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + # Empty/whitespace and 'nan' strings become NaN, NOT literal string 'nan' + self.assertTrue(pd.isna(processed.iloc[1, 3])) + self.assertTrue(pd.isna(processed.iloc[2, 1])) + self.assertTrue(pd.isna(processed.iloc[2, 2])) + # Numeric values preserved + self.assertEqual(processed.iloc[1, 1], 100) + self.assertEqual(processed.iloc[1, 2], 200.5) + + +class ReadsConfigFileTest(absltest.TestCase): + + def test_reads_local_config_file(self): + with tempfile.NamedTemporaryFile('w', suffix='.json', + delete=False) as f: + json.dump( + { + 'URLS_CONFIG': [{ + 'url': 'http://example.com/test.xlsx', + 'category': 'test', + 'filename': 'test.xlsx' + }] + }, f) + temp_path = f.name + try: + with flagsaver.flagsaver(config_file_path=temp_path): + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + self.assertEqual(len(configs['URLS_CONFIG']), 1) + self.assertEqual(configs['URLS_CONFIG'][0]['filename'], + 'test.xlsx') + finally: + os.remove(temp_path) + + def test_falls_back_to_local_configs_json(self): + with flagsaver.flagsaver(config_file_path='gs://nonexistent_bucket/configs.json'), \ + mock.patch.object(rbi_download.storage, 'Client', side_effect=Exception('GCS unavailable')): + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + self.assertGreater(len(configs['URLS_CONFIG']), 0) + + +class DownloadFilesTest(absltest.TestCase): + + def test_skips_existing_valid_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + cat_dir = os.path.join(temp_dir, 'agri') + os.makedirs(cat_dir, exist_ok=True) + existing_file = os.path.join(cat_dir, 'existing.xlsx') + with open(existing_file, 'wb') as f: + f.write(b'PK\x03\x04valid_zip_content') + + configs = [{ + 'url': 'http://example.com/existing.xlsx', + 'category': 'agri', + 'filename': 'existing.xlsx' + }] + mock_session = mock.MagicMock() + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + mock_session.get.assert_not_called() + + def test_redownloads_invalid_existing_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + cat_dir = os.path.join(temp_dir, 'agri') + os.makedirs(cat_dir, exist_ok=True) + corrupt_file = os.path.join(cat_dir, 'corrupt.xlsx') + with open(corrupt_file, 'wb') as f: + f.write(b'error page') + + configs = [{ + 'url': 'http://example.com/corrupt.xlsx', + 'category': 'agri', + 'filename': 'corrupt.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.content = b'PK\x03\x04valid_content' + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + mock_session.get.assert_called_once() + self.assertEqual( + pathlib.Path(corrupt_file).read_bytes(), + b'PK\x03\x04valid_content') + + def test_download_files_uses_browser_headers_in_session(self): + session = rbi_download.create_retry_session() + headers = session.headers + self.assertIn('User-Agent', headers) + self.assertIn('Mozilla', headers['User-Agent']) + + def test_rejects_html_error_response(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [{ + 'url': 'http://example.com/blocked.xlsx', + 'category': 'agri', + 'filename': 'blocked.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.content = b'Access Denied' + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + target_file = pathlib.Path(temp_dir) / 'agri' / 'blocked.xlsx' + self.assertFalse(target_file.exists()) + + def test_isolates_download_failures(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [ + { + 'url': 'http://example.com/fail.xlsx', + 'category': 'agri', + 'filename': 'fail.xlsx' + }, + { + 'url': 'http://example.com/success.xlsx', + 'category': 'agri', + 'filename': 'success.xlsx' + }, + ] + mock_session = mock.MagicMock() + mock_success = mock.MagicMock() + mock_success.status_code = 200 + mock_success.content = b'PK\x03\x04success content' + mock_session.get.side_effect = [ + requests.exceptions.RequestException('404 Not Found'), + mock_success + ] + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + + success_file = pathlib.Path(temp_dir) / 'agri' / 'success.xlsx' + self.assertTrue(success_file.exists()) + self.assertEqual(success_file.read_bytes(), + b'PK\x03\x04success content') + if __name__ == '__main__': absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index 7f3c28c3bd..ed2aea2d1f 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -40,8 +40,8 @@ ALL INDIA,country/IND All-India,country/IND All India,country/IND Ladakh,wikidataId/Q200667 -Kolkata**,wikidataId/Q1348 -Chennai**,wikidataId/Q1352 +Kolkata,wikidataId/Q1348 +Chennai,wikidataId/Q1352 Mumbai,wikidataId/Q1156 TamilNadu,wikidataId/Q1445 Andaman & NicobarIslands,wikidataId/Q40888 \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index 29f9bdd623..f73ef6e360 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -2,43 +2,52 @@ set -e SCRIPT_PATH=$(realpath "$(dirname "$0")") +COUNTERS_PATH=$SCRIPT_PATH/counters +mkdir -p "$COUNTERS_PATH" -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/agriculture/*.xlsx --pv_map=$SCRIPT_PATH/agriculture_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/agriculture/agriculture_output || { echo "Error: Processing Agriculture files failed!"; exit 1; } +# Existing statvar MCF path with local fallback +LOCAL_MCF="$SCRIPT_PATH/stat_vars.mcf" +if [ -f "$LOCAL_MCF" ]; then + EXISTING_MCF="$LOCAL_MCF" +else + EXISTING_MCF="${EXISTING_STATVAR_MCF:-gs://unresolved_mcf/scripts/statvar/stat_vars.mcf}" +fi -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/agriculture/*.xlsx" --pv_map=$SCRIPT_PATH/agriculture_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/agriculture_counters.csv --output_path=$SCRIPT_PATH/output_files/agriculture/agriculture_output || { echo "Error: Processing Agriculture files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_forest_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_tree_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/sub_division_wise_annual_rainfall_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/environment_sdg_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/price_and_wages/*.xlsx --pv_map=$SCRIPT_PATH/price_wages_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output || { echo "Error: Processing price_and_wages files failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/environment_sdg_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_sdg_score_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/price_and_wages/*.xlsx" --pv_map=$SCRIPT_PATH/price_wages_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/price_and_wages_counters.csv --output_path=$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output || { echo "Error: Processing price_and_wages files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_per_capita_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_installed_capacity_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_power_requirement_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_national_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_railway_route_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_roads_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output || { echo "Error: Processing State-wise Length of Roads failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_state_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_electricity_transmission_distribution_losses_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_telephones_per_100_population_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } + +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } echo "All processing steps completed successfully." exit 0 - diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index c391694e7c..9655dc9f61 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -3,11 +3,20 @@ "rules": [ { "rule_id": "check_deleted_records_percent", - "description": "Checks that the percentage of deleted points is within the threshold.", + "description": "Checks that the percentage of deleted records is within the 7% threshold.", "validator": "DELETED_RECORDS_PERCENT", "params": { "threshold": 7 } + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT CASE WHEN any_value(StatVar) IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN any_value(StatVar) IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats", + "condition": "is_fresh = TRUE AND row_cnt > 0" + } } ] } From 263c3473084e698a1312692b9b77a9ca84c40e09 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 10 Sep 2026 10:58:36 +0000 Subject: [PATCH 04/20] Changes made as per comments --- .../india_rbi_state_statistics/README.md | 4 +- .../india_rbi_state_statistics/manifest.json | 14 ++---- .../rbi_download.py | 49 ++++++++++++++----- .../rbi_download_test.py | 40 +++++++++++++-- .../validation_config.json | 4 +- 5 files changed, 82 insertions(+), 29 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 96a71547f9..175b1330ac 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -294,7 +294,7 @@ python3 stat_var_processor.py \ ## Validation Configuration and Thresholds -This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). +This import uses `validation_config.json` with a 6% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. -- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/19k3zyNk_KR9D4dVpXdGvlXKk6AMOzoYU2I1khJSsX8w/edit?resourcekey=0-JR3lDK9eNfo3ftEV6YpGxQ&tab=t.0#heading=h.iiqodx2yoqc8). +- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index d7f96314e7..4cc195e48b 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -26,28 +26,24 @@ }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_sdg_score_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 7d6da73694..534bcb48b3 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -24,12 +24,8 @@ import pandas as pd import requests from requests.adapters import HTTPAdapter -import urllib3 from urllib3.util import Retry -# Suppress unverified HTTPS warnings for RBI endpoints -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) INPUT_DIR = os.path.join(SCRIPT_DIR, "input_files") DEFAULT_CONFIG_JSON = os.path.join(SCRIPT_DIR, 'configs.json') @@ -70,7 +66,6 @@ def create_retry_session( session.mount("https://", adapter) session.mount("http://", adapter) session.headers.update(DEFAULT_HEADERS) - session.verify = False return session @@ -140,11 +135,13 @@ def reads_config_file(): def download_files(URL_CONFIG, session=None, delay=0.5): if not URL_CONFIG: logging.warning("No URL configurations provided to download.") - return + return [] if session is None: session = create_retry_session() + failed_downloads = [] + for config in URL_CONFIG: config_url = config.get("url") category_name = config.get("category") @@ -187,6 +184,7 @@ def download_files(URL_CONFIG, session=None, delay=0.5): logging.error( f"Empty response received for {file_name} from {config_url}" ) + failed_downloads.append(file_name) continue if not content.startswith(XLSX_ZIP_SIGNATURE): @@ -194,6 +192,7 @@ def download_files(URL_CONFIG, session=None, delay=0.5): logging.error( f"Invalid file format received for {file_name} from {config_url}: " f"expected XLSX ZIP signature, got: {snippet!r}") + failed_downloads.append(file_name) continue # Write atomically to a temporary file, then move into place @@ -214,10 +213,15 @@ def download_files(URL_CONFIG, session=None, delay=0.5): except Exception as e: logging.error( f"Failed to download table {file_name} from {config_url}: {e}") + failed_downloads.append(file_name) + + return failed_downloads def _apply_map(df, func): - """Version-agnostic element-wise DataFrame mapping (uses df.map if available, else df.applymap).""" + """Version-agnostic element-wise DataFrame mapping + (uses df.map if available, else df.applymap). + """ if hasattr(df, 'map'): return df.map(func) return df.applymap(func) @@ -226,19 +230,21 @@ def _apply_map(df, func): def preprocess_files(directory_path): if not os.path.isdir(directory_path): logging.fatal(f"Error: Directory not found at '{directory_path}'") - return + return [directory_path] xlsx_files = [f for f in os.listdir(directory_path) if f.endswith('.xlsx')] if not xlsx_files: logging.fatal( f"No XLSX files found in the directory: {directory_path}") - return + return [directory_path] logging.info( f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'." ) + failed_files = [] + for file_name in sorted(xlsx_files): file_path = os.path.join(directory_path, file_name) logging.info(f"Processing file: {file_path}") @@ -308,20 +314,41 @@ def is_state_header(val): except Exception as e: logging.error(f"Error processing {file_name}: {e}") + failed_files.append(file_name) + + if failed_files: + logging.fatal( + f"Failed to process {len(failed_files)} file(s) in '{directory_path}': {failed_files}" + ) + return failed_files logging.info("All specified XLSX files have been processed.") + return [] def main(_): configs = reads_config_file() RBI_URL = configs['URLS_CONFIG'] - download_files(RBI_URL) + failed_downloads = download_files(RBI_URL) + if failed_downloads: + logging.fatal( + f"Download failed for {len(failed_downloads)} table(s): {failed_downloads}" + ) + return logging.info("Download process Completed successfully") directories = [ 'agriculture', 'environment', 'infrastructure', 'price_and_wages' ] + failed_preprocesses = [] for directory in directories: - preprocess_files(os.path.join(INPUT_DIR, directory)) + failed = preprocess_files(os.path.join(INPUT_DIR, directory)) + if failed: + failed_preprocesses.extend(failed) + if failed_preprocesses: + logging.fatal( + f"Preprocessing failed for {len(failed_preprocesses)} file(s): {failed_preprocesses}" + ) + return logging.info("Pre-process Completed successfully") diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index 15cce4876f..50f548fed7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -43,12 +43,16 @@ def test_preserves_workbook_and_original_error(self): pd.DataFrame, 'map', side_effect=ValueError('transform failed')), \ - mock.patch.object(rbi_download.logging, 'error') as mock_error: - rbi_download.preprocess_files(directory) + mock.patch.object(rbi_download.logging, 'error') as mock_error, \ + mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: + failed = rbi_download.preprocess_files(directory) self.assertEqual(workbook.read_bytes(), b'original workbook') mock_error.assert_called_once_with( 'Error processing source.xlsx: transform failed') + mock_fatal.assert_called_once() + self.assertIn('source.xlsx', mock_fatal.call_args[0][0]) + self.assertEqual(failed, ['source.xlsx']) def test_preprocess_files_converts_numeric_headers(self): with tempfile.TemporaryDirectory() as directory: @@ -288,14 +292,40 @@ def test_isolates_download_failures(self): ] with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): - rbi_download.download_files(configs, - session=mock_session, - delay=0) + failed = rbi_download.download_files(configs, + session=mock_session, + delay=0) success_file = pathlib.Path(temp_dir) / 'agri' / 'success.xlsx' self.assertTrue(success_file.exists()) self.assertEqual(success_file.read_bytes(), b'PK\x03\x04success content') + self.assertEqual(failed, ['fail.xlsx']) + + +class MainExecutionTest(absltest.TestCase): + + def test_main_fatal_when_download_fails(self): + with mock.patch.object( + rbi_download, 'reads_config_file', return_value={'URLS_CONFIG': []}), \ + mock.patch.object( + rbi_download, 'download_files', return_value=['failed_table.xlsx']), \ + mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: + rbi_download.main([]) + mock_fatal.assert_called_once() + self.assertIn('failed_table.xlsx', mock_fatal.call_args[0][0]) + + def test_main_fatal_when_preprocessing_fails(self): + with mock.patch.object( + rbi_download, 'reads_config_file', return_value={'URLS_CONFIG': []}), \ + mock.patch.object( + rbi_download, 'download_files', return_value=[]), \ + mock.patch.object( + rbi_download, 'preprocess_files', side_effect=[['broken.xlsx'], [], [], []]), \ + mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: + rbi_download.main([]) + mock_fatal.assert_called_once() + self.assertIn('broken.xlsx', mock_fatal.call_args[0][0]) if __name__ == '__main__': diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index 9655dc9f61..a44326b54c 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -14,9 +14,9 @@ "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT CASE WHEN any_value(StatVar) IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN any_value(StatVar) IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats", + "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats GROUP BY StatVar", "condition": "is_fresh = TRUE AND row_cnt > 0" } } ] -} +} \ No newline at end of file From 4e6de3e9a1a1b08693825aa0da18a1de7cf0b94d Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 11 Sep 2026 05:26:59 +0000 Subject: [PATCH 05/20] Made changes in downlaod script --- .../india_rbi_state_statistics/README.md | 2 +- .../india_rbi_state_statistics/manifest.json | 15 ++++--- .../rbi_download.py | 11 +++++- .../rbi_download_test.py | 39 +++++++++++++++++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 175b1330ac..9d63f0ba7e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -294,7 +294,7 @@ python3 stat_var_processor.py \ ## Validation Configuration and Thresholds -This import uses `validation_config.json` with a 6% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). +This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. - **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 4cc195e48b..b0fb58dfcf 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -16,7 +16,8 @@ "input_files/environment/*.xlsx", "input_files/infrastructure/*.xlsx", "input_files/price_and_wages/*.xlsx", - "counters/*.csv" + "counters/*.csv", + "configs.json" ], "import_inputs": [ { @@ -26,11 +27,13 @@ }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", @@ -39,11 +42,13 @@ }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 534bcb48b3..d6c37e3e5c 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -111,6 +111,12 @@ def reads_config_file(): file_contents = blob.download_as_text() if config_file_path.endswith('.json'): return json.loads(file_contents) + # Deprecated: Legacy Python config (.py) via exec(). + # All imports should standardize on version-controlled JSON configuration. + logging.warning( + f"DEPRECATION WARNING: Loading Python configuration via exec() " + f"from '{config_file_path}' is deprecated and will be removed. " + "Please migrate to JSON format ('configs.json').") local_vars = {} exec(file_contents, {}, local_vars) return local_vars @@ -172,9 +178,10 @@ def download_files(URL_CONFIG, session=None, delay=0.5): response = session.get(config_url, timeout=45) if response.status_code == 404: - logging.warning( - f"Table not found (HTTP 404) at {config_url}, skipping {file_name}" + logging.error( + f"Table not found (HTTP 404) at {config_url} for {file_name}" ) + failed_downloads.append(file_name) continue response.raise_for_status() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index 50f548fed7..ad086458d8 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -191,6 +191,25 @@ def test_falls_back_to_local_configs_json(self): self.assertIn('URLS_CONFIG', configs) self.assertGreater(len(configs['URLS_CONFIG']), 0) + def test_reads_gcs_python_config_logs_deprecation(self): + mock_blob = mock.MagicMock() + mock_blob.download_as_text.return_value = 'URLS_CONFIG = [{"url": "http://example.com/test.xlsx", "category": "cat", "filename": "test.xlsx"}]\n' + mock_bucket = mock.MagicMock() + mock_bucket.blob.return_value = mock_blob + mock_client = mock.MagicMock() + mock_client.bucket.return_value = mock_bucket + + with flagsaver.flagsaver(config_file_path='gs://bucket/legacy_configs.py'), \ + mock.patch.object(rbi_download.storage, 'Client', return_value=mock_client), \ + mock.patch.object(rbi_download.logging, 'warning') as mock_warning: + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + deprecation_logged = any( + 'DEPRECATION WARNING' in str(call[0][0]) + for call in mock_warning.call_args_list + ) + self.assertTrue(deprecation_logged) + class DownloadFilesTest(absltest.TestCase): @@ -268,6 +287,26 @@ def test_rejects_html_error_response(self): target_file = pathlib.Path(temp_dir) / 'agri' / 'blocked.xlsx' self.assertFalse(target_file.exists()) + def test_treats_404_as_failure(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [{ + 'url': 'http://example.com/missing.xlsx', + 'category': 'agri', + 'filename': 'missing.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 404 + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + failed = rbi_download.download_files(configs, + session=mock_session, + delay=0) + self.assertEqual(failed, ['missing.xlsx']) + target_file = pathlib.Path(temp_dir) / 'agri' / 'missing.xlsx' + self.assertFalse(target_file.exists()) + def test_isolates_download_failures(self): with tempfile.TemporaryDirectory() as temp_dir: configs = [ From fa01b14f2ec536f8d403be09e2b7ff2da41e5838 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 11 Sep 2026 06:44:26 +0000 Subject: [PATCH 06/20] Modified naming in config.json --- .../india_rbi_state_statistics/configs.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json index 52ef784515..11f90b347e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json @@ -313,17 +313,17 @@ { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/112T_11122025771E55EB688044CE8A85E52A901B5D22.XLSX", "category": "price_and_wages", - "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_construction_workers.xlsx" + "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_construction_workers.xlsx" }, { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/113T_111220256FEBCC253D4A47419892925F897E2537.XLSX", "category": "price_and_wages", - "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_general_agricultural_labourers.xlsx" + "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_general_agricultural_labourers.xlsx" }, { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/114T_111220259CA566982E7F4BF38B3FDAC0D7587F2E.XLSX", "category": "price_and_wages", - "filename": "state_wise_Average Daily Wage Rates in Rural_india_men_horticulture_workers.xlsx" + "filename": "state_wise_Average_daily_wage_rates_in_rural_india_men_horticulture_workers.xlsx" }, { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/115T_1112202585CF99F21E654CED8990A734A043B3DA.XLSX", From 26ae732fc243642ef30bf7a331429434cc6cdca2 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Tue, 15 Sep 2026 08:04:37 +0000 Subject: [PATCH 07/20] Resolved comments --- .../environment_pvmap.csv | 2 +- .../india_rbi_state_statistics/rbi_download.py | 14 ++++++++++++++ .../rbi_download_test.py | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index ebd44f700d..8afbad64f9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -6,7 +6,7 @@ Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProper Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, -"Haryana, Delhi & Chandigarh",observationAbout,country/IND +"Haryana, Delhi & Chandigarh",observationAbout,dcid:HaryanaDelhiChandigarh 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index d6c37e3e5c..74ad581287 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -41,8 +41,14 @@ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', + 'Referer': + 'https://www.rbi.org.in/', } +RBI_PORTAL_URL = ( + 'https://www.rbi.org.in/Scripts/AnnualPublications.aspx?' + 'head=Handbook%20of%20Statistics%20on%20Indian%20States' +) XLSX_ZIP_SIGNATURE = b'PK\x03\x04' @@ -145,6 +151,14 @@ def download_files(URL_CONFIG, session=None, delay=0.5): if session is None: session = create_retry_session() + try: + logging.info( + f"Warming up session with RBI portal cookies from {RBI_PORTAL_URL}" + ) + session.get(RBI_PORTAL_URL, timeout=30) + except Exception as e: + logging.warning( + f"Could not warm up session from {RBI_PORTAL_URL}: {e}") failed_downloads = [] diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index ad086458d8..1b9b229020 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -266,6 +266,8 @@ def test_download_files_uses_browser_headers_in_session(self): headers = session.headers self.assertIn('User-Agent', headers) self.assertIn('Mozilla', headers['User-Agent']) + self.assertIn('Referer', headers) + self.assertIn('rbi.org.in', headers['Referer']) def test_rejects_html_error_response(self): with tempfile.TemporaryDirectory() as temp_dir: From 9d291ed728514de0e5a1add2c6fa8fd328f8cd73 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 17 Sep 2026 08:07:08 +0000 Subject: [PATCH 08/20] CRA analysis done --- .../india_rbi_state_statistics/README.md | 102 ++++++++++++------ .../india_rbi_state_statistics/manifest.json | 34 +++--- .../rbi_download.py | 6 +- .../india_rbi_state_statistics/run.sh | 3 +- .../validation_config.json | 11 +- 5 files changed, 103 insertions(+), 53 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 9d63f0ba7e..e56ba2a9db 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -29,14 +29,16 @@ This makes the import process **Semi-Automated**: if download URLs change in fut ## Data Acquisition and Initial Preprocessing -The `rbi_download.py` script is responsible for both downloading the raw Excel data and performing an initial preprocessing step. This preprocessing involves reading all sheets from the downloaded `.xlsx` files and replacing any `*` symbols with empty values. +The `rbi_download.py` script is responsible for both downloading the raw Excel data and performing an initial preprocessing step: +- **Cell Cleaning (`clean_cell`)**: Strips footnote markers (`*` and `@`), trims leading/trailing whitespace, and converts empty or literal `'nan'` strings to true `NaN` values so `stat_var_processor.py` does not emit malformed string observations. +- **Header Normalization (`safe_to_numeric`)**: Converts numeric or float-coerced year headers (e.g., `2015.0`, `'2016@'`) in `State/Union Territory` header rows to integers (`2015`, `2016`) so they match integer string keys in the PVMaps. ### How to Run: Execute the `rbi_download.py` script. This script will: 1. Automatically create an `input_files` directory if it doesn't exist. -2. Download the necessary Excel files into subfolders within `input_files` (e.g., `input_files/agriculture/`). -3. Process each downloaded Excel file by reading all its sheets and replacing `*` characters with empty strings in all cells. +2. Download the necessary Excel files into subfolders within `input_files` (e.g., `input_files/agriculture/`), verifying `PK\x03\x04` ZIP signatures and writing files atomically. +3. Process each downloaded Excel file across all sheets using `clean_cell` and `safe_to_numeric`. ### Download Command: @@ -44,20 +46,32 @@ Execute the `rbi_download.py` script. This script will: python3 rbi_download.py ``` +Optional flag: +- `--config_file_path`: Path to the JSON config file (defaults to local `configs.json`; also supports a GCS URI such as `gs:///configs.json`). + +### Running Unit Tests: + +To run the hermetic unit test suite for `rbi_download.py`: + +```bash +python3 rbi_download_test.py +``` + ## Processing Section -The downloaded data is processed using the `stat_var_processor.py` script, which is part of the `/data/tools/statvar_importer/` toolkit. This script converts the raw Excel data into a structured format suitable for further analysis and ingestion. Each processing command specifies the input data file(s), the Property-Value (PV) map, the configuration file (metadata), a places resolver CSV, and the desired output path. +The downloaded data is processed using the `stat_var_processor.py` script, which is part of the `/data/tools/statvar_importer/` toolkit. This script converts the raw Excel data into a structured format suitable for further analysis and ingestion. Each processing command specifies the input data file(s), the Property-Value (PV) map, the configuration file (metadata), a places resolver CSV, counter output CSV, and the desired output path. ### General Processing Command Structure: ```bash python3 stat_var_processor.py \ - --input_data='.xlsx' \ - --pv_map='' \ - --config_file='' \ - --places_resolved_csv='' \ - --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ - --output_path='' + --input_data=".xlsx" \ + --pv_map="" \ + --config_file="" \ + --places_resolved_csv="" \ + --existing_statvar_mcf="gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" \ + --output_counters="" \ + --output_path="" ``` ### How to Run Processing: @@ -72,7 +86,7 @@ sh run.sh #### Option 2: Executing Individual Processing Commands -Navigate to the `/data/tools/statvar_importer/` directory before running the following commands, or adjust the relative paths accordingly. +Navigate to the `/data/tools/statvar_importer/` directory before running the following commands, or adjust the relative paths accordingly. Note that wildcard `--input_data` paths must be enclosed in double quotes so `stat_var_processor.py` expands them internally. ### Processing Commands: @@ -80,11 +94,12 @@ Navigate to the `/data/tools/statvar_importer/` directory before running the fol ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/agriculture/*.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/agriculture/*.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/agriculture_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/agriculture_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/agriculture/agriculture_output ``` @@ -92,11 +107,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_forest_cover.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_forest_cover.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_forest_cover_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_forest_cover_output ``` @@ -104,11 +120,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_tree_cover.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_tree_cover.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_tree_cover_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_tree_cover_output ``` @@ -116,11 +133,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/sub_division_wise_annual_rainfall.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/sub_division_wise_annual_rainfall.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/sub_division_wise_annual_rainfall_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/sub_division_wise_annual_rainfall_output ``` @@ -128,11 +146,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output ``` @@ -140,11 +159,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_sdg_score_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_sdg_score_output ``` @@ -152,11 +172,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/price_and_wages/*.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/price_and_wages/*.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/price_wages_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/price_and_wages_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/price_and_wages/price_and_wages_output ``` @@ -164,11 +185,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_per_capita_availability_of_power_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_per_capita_availability_of_power_output ``` @@ -176,11 +198,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_availability_of_power.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_availability_of_power.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_availability_of_power_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_availability_of_power_output ``` @@ -188,11 +211,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_installed_capacity_of_power_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_installed_capacity_of_power_output ``` @@ -200,11 +224,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_power_requirement.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_power_requirement.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_power_requirement_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_power_requirement_output ``` @@ -212,11 +237,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_national_highways.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_length_of_national_highways_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_length_of_national_highways_output ``` @@ -224,11 +250,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_railway_route.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_railway_route.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_railway_route_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_railway_route_output ``` @@ -236,11 +263,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_roads.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_roads.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_length_of_roads_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_length_of_roads_output ``` @@ -248,11 +276,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_state_highways.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_length_of_state_highways_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_length_of_state_highways_output ``` @@ -260,11 +289,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_electricity_transmission_distribution_losses_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output ``` @@ -272,11 +302,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_telephones_per_100_population_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_telephones_per_100_population_output ``` @@ -284,17 +315,24 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_road_constructed_under_pmgsy_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output ``` ## Validation Configuration and Thresholds -This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). -- **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. -- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). +This import uses `validation_config.json` with: +1. **`DELETED_RECORDS_PERCENT: 7` (`check_deleted_records_percent`)**: + - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. + - **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). +2. **`SQL_VALIDATOR` non-empty check (`check_expected_statvar_count`)**: + - Asserts `statvar_cnt >= 1` so an empty `summary_report.csv` cannot vacuously pass grouped SQL queries. +3. **`SQL_VALIDATOR` cadence freshness check (`check_max_date_freshness`)**: + - Asserts that each StatVar meets its expected RBI publication cadence (`2020`, `2022`, `2023`, or `2024`) with `WHERE MaxDate IS NOT NULL`. + diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index b0fb58dfcf..7b14fed2a4 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -23,17 +23,17 @@ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", "cleaned_csv": "output_files/agriculture/agriculture_output.csv", - "node_mcf": "output_files/agriculture/*.mcf" + "node_mcf": "output_files/agriculture/agriculture_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_forest_cover_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", @@ -43,72 +43,72 @@ { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_tree_cover_output*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", - "node_mcf": "output_files/price_and_wages/*.mcf" + "node_mcf": "output_files/price_and_wages/price_and_wages_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_availability_of_power_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_length_of_roads_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_power_requirement_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_railway_route_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output*.mcf" } ], "cron_schedule": "0 10 * * 1", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 74ad581287..2868fb801a 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -14,9 +14,7 @@ import json import os -import pathlib import re -import sys import tempfile import time from absl import app, flags, logging @@ -72,6 +70,7 @@ def create_retry_session( session.mount("https://", adapter) session.mount("http://", adapter) session.headers.update(DEFAULT_HEADERS) + session.verify = True return session @@ -332,6 +331,9 @@ def is_state_header(val): index=False, header=False) os.replace(output_path, file_path) + logging.info( + f"Successfully processed {file_name} ({len(all_sheets_data)} sheet(s))." + ) except Exception as e: logging.error(f"Error processing {file_name}: {e}") diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index f73ef6e360..5415a0d12b 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -49,5 +49,6 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } +EXIT_STATUS=$? echo "All processing steps completed successfully." -exit 0 +exit $EXIT_STATUS diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index a44326b54c..dfed4dc110 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -9,12 +9,21 @@ "threshold": 7 } }, + { + "rule_id": "check_expected_statvar_count", + "description": "Verifies that summary_report.csv is non-empty and contains at least one StatVar.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT COUNT(DISTINCT StatVar) AS statvar_cnt FROM stats", + "condition": "statvar_cnt >= 1" + } + }, { "rule_id": "check_max_date_freshness", "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats GROUP BY StatVar", + "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats WHERE MaxDate IS NOT NULL GROUP BY StatVar", "condition": "is_fresh = TRUE AND row_cnt > 0" } } From aed61388befbcd66ac9a5f0ce66a7bf3bb27dd77 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 17 Sep 2026 14:15:23 +0000 Subject: [PATCH 09/20] Changes as per Internal reviews --- .../india_rbi_state_statistics/README.md | 5 +- .../india_rbi_state_statistics/configs.json | 2 +- .../india_rbi_state_statistics/manifest.json | 5 +- .../rbi_download.py | 18 +++++-- .../rbi_download_test.py | 51 ++++++++++++------- .../rbi_places_resolver.csv | 2 +- .../validation_config.json | 4 +- 7 files changed, 56 insertions(+), 31 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index e56ba2a9db..a4bb6e46aa 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -328,7 +328,10 @@ python3 stat_var_processor.py \ This import uses `validation_config.json` with: 1. **`DELETED_RECORDS_PERCENT: 7` (`check_deleted_records_percent`)**: - - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. + - **Rationale & Analysis**: The threshold is set to `7%` to accommodate up to **6.94%** reported deletions across the 18 sub-imports (specifically **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, and **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`). These deletions are not data loss, but result from: + 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q9357528` (439 records in agriculture, 13 records in relief expenditure). + 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, where Import Differ classifies old unit nodes as deleted and new unit nodes as added). + 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations (`Dadra & Nagar Haveli and Daman & Diu`). - **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). 2. **`SQL_VALIDATOR` non-empty check (`check_expected_statvar_count`)**: - Asserts `statvar_cnt >= 1` so an empty `summary_report.csv` cannot vacuously pass grouped SQL queries. diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json index 11f90b347e..023961b31b 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json @@ -331,4 +331,4 @@ "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_non_agricultural_labourers.xlsx" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 7b14fed2a4..9642adcdb7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -17,7 +17,8 @@ "input_files/infrastructure/*.xlsx", "input_files/price_and_wages/*.xlsx", "counters/*.csv", - "configs.json" + "configs.json", + "validation_config.json" ], "import_inputs": [ { @@ -115,4 +116,4 @@ "validation_config_file": "validation_config.json" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 2868fb801a..2db01a66af 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -30,6 +30,8 @@ flags.DEFINE_string('config_file_path', DEFAULT_CONFIG_JSON, 'Config file path (local json or gs:// path)') +flags.DEFINE_bool('force_download', False, + 'If true, redownload files even if valid local copies exist.') DEFAULT_HEADERS = { 'User-Agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' @@ -173,14 +175,20 @@ def download_files(URL_CONFIG, session=None, delay=0.5): os.makedirs(target_dir, exist_ok=True) file_path = os.path.join(target_dir, file_name) + force_download = False + try: + force_download = flags.FLAGS.force_download + except (flags.UnparsedFlagAccessError, AttributeError): + pass + if os.path.exists(file_path): - if os.path.getsize(file_path) > 0: + if not force_download and os.path.getsize(file_path) > 0: with open(file_path, 'rb') as f: magic = f.read(4) if magic == XLSX_ZIP_SIGNATURE: logging.info(f"Skipping existing valid file: {file_name}") continue - # Remove stale, empty, or corrupt file + # Remove stale, empty, or corrupt file (or existing file if force_download) try: os.remove(file_path) except OSError: @@ -249,13 +257,13 @@ def _apply_map(df, func): def preprocess_files(directory_path): if not os.path.isdir(directory_path): - logging.fatal(f"Error: Directory not found at '{directory_path}'") + logging.error(f"Error: Directory not found at '{directory_path}'") return [directory_path] xlsx_files = [f for f in os.listdir(directory_path) if f.endswith('.xlsx')] if not xlsx_files: - logging.fatal( + logging.error( f"No XLSX files found in the directory: {directory_path}") return [directory_path] @@ -340,7 +348,7 @@ def is_state_header(val): failed_files.append(file_name) if failed_files: - logging.fatal( + logging.error( f"Failed to process {len(failed_files)} file(s) in '{directory_path}': {failed_files}" ) return failed_files diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index 1b9b229020..f672bb7423 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -48,10 +48,10 @@ def test_preserves_workbook_and_original_error(self): failed = rbi_download.preprocess_files(directory) self.assertEqual(workbook.read_bytes(), b'original workbook') - mock_error.assert_called_once_with( + mock_error.assert_any_call( 'Error processing source.xlsx: transform failed') - mock_fatal.assert_called_once() - self.assertIn('source.xlsx', mock_fatal.call_args[0][0]) + self.assertEqual(mock_error.call_count, 2) + mock_fatal.assert_not_called() self.assertEqual(failed, ['source.xlsx']) def test_preprocess_files_converts_numeric_headers(self): @@ -97,18 +97,21 @@ def test_preprocess_files_preserves_nan(self): self.assertTrue(pd.isna(processed.iloc[0, 2])) self.assertTrue(pd.isna(processed.iloc[1, 1])) - def test_preprocess_files_fatal_on_missing_dir(self): - with mock.patch.object(rbi_download.logging, 'fatal') as fatal: - rbi_download.preprocess_files('/non/existent/directory/path') - fatal.assert_called_once() - self.assertIn("Directory not found", fatal.call_args[0][0]) + def test_preprocess_files_error_on_missing_dir(self): + with mock.patch.object(rbi_download.logging, 'error') as mock_error: + failed = rbi_download.preprocess_files( + '/non/existent/directory/path') + self.assertEqual(failed, ['/non/existent/directory/path']) + mock_error.assert_called_once() + self.assertIn("Directory not found", mock_error.call_args[0][0]) - def test_preprocess_files_fatal_on_empty_dir(self): + def test_preprocess_files_error_on_empty_dir(self): with tempfile.TemporaryDirectory() as directory: - with mock.patch.object(rbi_download.logging, 'fatal') as fatal: - rbi_download.preprocess_files(directory) - fatal.assert_called_once() - self.assertIn("No XLSX files found", fatal.call_args[0][0]) + with mock.patch.object(rbi_download.logging, 'error') as mock_error: + failed = rbi_download.preprocess_files(directory) + self.assertEqual(failed, [directory]) + mock_error.assert_called_once() + self.assertIn("No XLSX files found", mock_error.call_args[0][0]) def test_preprocess_files_handles_flexible_state_header(self): with tempfile.TemporaryDirectory() as directory: @@ -185,23 +188,33 @@ def test_reads_local_config_file(self): os.remove(temp_path) def test_falls_back_to_local_configs_json(self): - with flagsaver.flagsaver(config_file_path='gs://nonexistent_bucket/configs.json'), \ - mock.patch.object(rbi_download.storage, 'Client', side_effect=Exception('GCS unavailable')): + with flagsaver.flagsaver( + config_file_path='gs://nonexistent_bucket/configs.json'), \ + mock.patch.object( + rbi_download.storage, + 'Client', + side_effect=Exception('GCS unavailable')): configs = rbi_download.reads_config_file() self.assertIn('URLS_CONFIG', configs) self.assertGreater(len(configs['URLS_CONFIG']), 0) def test_reads_gcs_python_config_logs_deprecation(self): mock_blob = mock.MagicMock() - mock_blob.download_as_text.return_value = 'URLS_CONFIG = [{"url": "http://example.com/test.xlsx", "category": "cat", "filename": "test.xlsx"}]\n' + mock_blob.download_as_text.return_value = ( + 'URLS_CONFIG = [{"url": "http://example.com/test.xlsx", ' + '"category": "cat", "filename": "test.xlsx"}]\n' + ) mock_bucket = mock.MagicMock() mock_bucket.blob.return_value = mock_blob mock_client = mock.MagicMock() mock_client.bucket.return_value = mock_bucket - with flagsaver.flagsaver(config_file_path='gs://bucket/legacy_configs.py'), \ - mock.patch.object(rbi_download.storage, 'Client', return_value=mock_client), \ - mock.patch.object(rbi_download.logging, 'warning') as mock_warning: + with flagsaver.flagsaver( + config_file_path='gs://bucket/legacy_configs.py'), \ + mock.patch.object( + rbi_download.storage, 'Client', return_value=mock_client), \ + mock.patch.object( + rbi_download.logging, 'warning') as mock_warning: configs = rbi_download.reads_config_file() self.assertIn('URLS_CONFIG', configs) deprecation_logged = any( diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index ed2aea2d1f..0aadea27af 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -44,4 +44,4 @@ Kolkata,wikidataId/Q1348 Chennai,wikidataId/Q1352 Mumbai,wikidataId/Q1156 TamilNadu,wikidataId/Q1445 -Andaman & NicobarIslands,wikidataId/Q40888 \ No newline at end of file +Andaman & NicobarIslands,wikidataId/Q40888 diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index dfed4dc110..fee5f45bb9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -3,7 +3,7 @@ "rules": [ { "rule_id": "check_deleted_records_percent", - "description": "Checks that the percentage of deleted records is within the 7% threshold.", + "description": "Checks that the percentage of deleted records is within the 7% threshold. Justification: Accommodates up to 6.94% deletions observed in sub-imports (6.94% in sub_division_wise_annual_rainfall and 5.64% in agriculture) due to: (1) place resolution fixes remapping 'Haryana, Delhi & Chandigarh' from Q1174 to dcid:HaryanaDelhiChandigarh and 'NCT of Delhi' from Q1352 (Chennai) to Q9357528, (2) unit correction for Annual_Amount_FarmInventory_Fruits from Hectare to MetricTon, and (3) official RBI retrospective revisions to provisional historical data.", "validator": "DELETED_RECORDS_PERCENT", "params": { "threshold": 7 @@ -28,4 +28,4 @@ } } ] -} \ No newline at end of file +} From 5cfdddd4a45a79896cd3e20b58c5565bdd9b94ac Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 21 Sep 2026 11:29:27 +0000 Subject: [PATCH 10/20] CRA based changes --- .../india_rbi_state_statistics/README.md | 28 +- .../environment_pvmap.csv | 1 + .../rbi_download.py | 29 +- .../rbi_download_test.py | 9 +- .../testdata/environment_output.csv | 656 ++++++++++++++++++ 5 files changed, 704 insertions(+), 19 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index a4bb6e46aa..9c74cd9d80 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -1,4 +1,3 @@ - # India_RBI_State_Statistics ## Import Overview @@ -57,6 +56,20 @@ To run the hermetic unit test suite for `rbi_download.py`: python3 rbi_download_test.py ``` +## Prerequisites + +The following Python packages are required to run the download, preprocessing, and test suites: +- `pandas` +- `openpyxl` +- `requests` +- `google-cloud-storage` +- `absl-py` + +Install dependencies via: +```bash +pip install pandas openpyxl requests google-cloud-storage absl-py +``` + ## Processing Section The downloaded data is processed using the `stat_var_processor.py` script, which is part of the `/data/tools/statvar_importer/` toolkit. This script converts the raw Excel data into a structured format suitable for further analysis and ingestion. Each processing command specifies the input data file(s), the Property-Value (PV) map, the configuration file (metadata), a places resolver CSV, counter output CSV, and the desired output path. @@ -338,4 +351,17 @@ This import uses `validation_config.json` with: 3. **`SQL_VALIDATOR` cadence freshness check (`check_max_date_freshness`)**: - Asserts that each StatVar meets its expected RBI publication cadence (`2020`, `2022`, `2023`, or `2024`) with `WHERE MaxDate IS NOT NULL`. +## Refresh & Operational Maintenance + +The import is configured with an automated cron schedule in `manifest.json`: +- **Schedule:** `0 10 * * 1` (Every Monday at 10:00 AM UTC). +- **Import Type:** Semi-Automated. + +### Annual Refresh Procedure: +1. When the Reserve Bank of India publishes a new edition of the *Handbook of Statistics on Indian States*, check the [RBI Annual Publications Portal](https://www.rbi.org.in/Scripts/AnnualPublications.aspx?head=Handbook%20of%20Statistics%20on%20Indian%20States). +2. If table download URLs have changed, update the `url` fields in `configs.json`. +3. If new tables, columns, or indicators are introduced, update the corresponding PVMap (`agriculture_pvmap.csv`, `environment_pvmap.csv`, `infrastructure_pvmap.csv`, or `price_wages_pvmap.csv`). +4. Re-run `python3 rbi_download.py` followed by `sh run.sh` to download and process the updated datasets. +5. Verify test outputs and pre-submit validation before raising a pull request. + diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 8afbad64f9..35e1c04fc6 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -7,6 +7,7 @@ Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredPropert State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, "Haryana, Delhi & Chandigarh",observationAbout,dcid:HaryanaDelhiChandigarh +"Haryana, Delhi and Chandigarh",observationAbout,dcid:HaryanaDelhiChandigarh 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 2db01a66af..206ace20c9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -169,6 +169,7 @@ def download_files(URL_CONFIG, session=None, delay=0.5): file_name = config.get("filename") if not config_url or not category_name or not file_name: logging.warning(f"Skipping incomplete config: {config}") + failed_downloads.append(file_name or str(config)) continue target_dir = os.path.join(INPUT_DIR, category_name) @@ -188,11 +189,11 @@ def download_files(URL_CONFIG, session=None, delay=0.5): if magic == XLSX_ZIP_SIGNATURE: logging.info(f"Skipping existing valid file: {file_name}") continue - # Remove stale, empty, or corrupt file (or existing file if force_download) - try: - os.remove(file_path) - except OSError: - pass + if not force_download and os.path.getsize(file_path) == 0: + try: + os.remove(file_path) + except OSError: + pass try: logging.info(f"Attempting GET request to: {config_url}") @@ -223,14 +224,12 @@ def download_files(URL_CONFIG, session=None, delay=0.5): failed_downloads.append(file_name) continue - # Write atomically to a temporary file, then move into place - with tempfile.NamedTemporaryFile('wb', - dir=target_dir, - delete=False) as tmp_file: - tmp_file.write(content) - temp_path = tmp_file.name - - os.replace(temp_path, file_path) + # Write atomically to a temporary directory, then move into place + with tempfile.TemporaryDirectory(dir=target_dir) as tmp_dir: + temp_path = os.path.join(tmp_dir, file_name) + with open(temp_path, 'wb') as tmp_file: + tmp_file.write(content) + os.replace(temp_path, file_path) logging.info( f"Downloaded the file {file_name} successfully ({len(content)} bytes)." ) @@ -295,8 +294,10 @@ def clean_cell(val): def safe_to_numeric(val): if pd.isna(val): return val - if isinstance(val, (int, float)): + if isinstance(val, int) and not isinstance(val, bool): return val + if isinstance(val, float): + return int(val) if val.is_integer() else val val_str = str(val).strip() if not val_str or val_str.lower() == 'nan': return float('nan') diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index f672bb7423..ce1be2c66c 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -40,8 +40,8 @@ def test_preserves_workbook_and_original_error(self): with mock.patch.object( rbi_download.pd, 'read_excel', return_value=sheets), \ mock.patch.object( - pd.DataFrame, - 'map', + rbi_download, + '_apply_map', side_effect=ValueError('transform failed')), \ mock.patch.object(rbi_download.logging, 'error') as mock_error, \ mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: @@ -58,8 +58,8 @@ def test_preprocess_files_converts_numeric_headers(self): with tempfile.TemporaryDirectory() as directory: file_path = pathlib.Path(directory) / 'source.xlsx' initial_df = pd.DataFrame( - [['State/Union Territory', '2015*', '2016@', '2017-18'], - ['Andhra Pradesh', '10.5', '20.0', '30.5']]) + [['State/Union Territory', '2015*', '2016@', '2017-18', 2018.0], + ['Andhra Pradesh', '10.5', '20.0', '30.5', '40.0']]) with pd.ExcelWriter(file_path, engine='openpyxl') as writer: initial_df.to_excel(writer, sheet_name='Sheet1', @@ -75,6 +75,7 @@ def test_preprocess_files_converts_numeric_headers(self): self.assertEqual(processed.iloc[0, 1], 2015) self.assertEqual(processed.iloc[0, 2], 2016) self.assertEqual(processed.iloc[0, 3], '2017-18') + self.assertEqual(processed.iloc[0, 4], 2018) self.assertEqual(processed.iloc[1, 0], 'Andhra Pradesh') def test_preprocess_files_preserves_nan(self): diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/testdata/environment_output.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/testdata/environment_output.csv index a03144fc2e..94b2198548 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/testdata/environment_output.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/testdata/environment_output.csv @@ -1 +1,657 @@ observationAbout,observationDate,variableMeasured,value,unit,observationPeriod +dcid:wikidataId/Q40888,1987,dcid:Area_LandCover_Forest,7601,SquareKilometer, +dcid:wikidataId/Q40888,1989,dcid:Area_LandCover_Forest,7622,SquareKilometer, +dcid:wikidataId/Q40888,1991,dcid:Area_LandCover_Forest,7622,SquareKilometer, +dcid:wikidataId/Q40888,1993,dcid:Area_LandCover_Forest,7624,SquareKilometer, +dcid:wikidataId/Q40888,1995,dcid:Area_LandCover_Forest,7615,SquareKilometer, +dcid:wikidataId/Q40888,1997,dcid:Area_LandCover_Forest,7613,SquareKilometer, +dcid:wikidataId/Q40888,1999,dcid:Area_LandCover_Forest,7606,SquareKilometer, +dcid:wikidataId/Q40888,2001,dcid:Area_LandCover_Forest,6930,SquareKilometer, +dcid:wikidataId/Q40888,2003,dcid:Area_LandCover_Forest,6964,SquareKilometer, +dcid:wikidataId/Q40888,2005,dcid:Area_LandCover_Forest,6663,SquareKilometer, +dcid:wikidataId/Q1159,1987,dcid:Area_LandCover_Forest,49573,SquareKilometer, +dcid:wikidataId/Q1159,1989,dcid:Area_LandCover_Forest,47290,SquareKilometer, +dcid:wikidataId/Q1159,1991,dcid:Area_LandCover_Forest,47290,SquareKilometer, +dcid:wikidataId/Q1159,1993,dcid:Area_LandCover_Forest,47256,SquareKilometer, +dcid:wikidataId/Q1159,1995,dcid:Area_LandCover_Forest,47112,SquareKilometer, +dcid:wikidataId/Q1159,1997,dcid:Area_LandCover_Forest,43290,SquareKilometer, +dcid:wikidataId/Q1159,1999,dcid:Area_LandCover_Forest,44229,SquareKilometer, +dcid:wikidataId/Q1159,2001,dcid:Area_LandCover_Forest,44637,SquareKilometer, +dcid:wikidataId/Q1159,2003,dcid:Area_LandCover_Forest,44419,SquareKilometer, +dcid:wikidataId/Q1159,2005,dcid:Area_LandCover_Forest,45231,SquareKilometer, +dcid:wikidataId/Q1162,1987,dcid:Area_LandCover_Forest,64132,SquareKilometer, +dcid:wikidataId/Q1162,1989,dcid:Area_LandCover_Forest,69002,SquareKilometer, +dcid:wikidataId/Q1162,1991,dcid:Area_LandCover_Forest,68757,SquareKilometer, +dcid:wikidataId/Q1162,1993,dcid:Area_LandCover_Forest,68661,SquareKilometer, +dcid:wikidataId/Q1162,1995,dcid:Area_LandCover_Forest,68621,SquareKilometer, +dcid:wikidataId/Q1162,1997,dcid:Area_LandCover_Forest,68602,SquareKilometer, +dcid:wikidataId/Q1162,1999,dcid:Area_LandCover_Forest,68847,SquareKilometer, +dcid:wikidataId/Q1162,2001,dcid:Area_LandCover_Forest,68045,SquareKilometer, +dcid:wikidataId/Q1162,2003,dcid:Area_LandCover_Forest,68019,SquareKilometer, +dcid:wikidataId/Q1162,2005,dcid:Area_LandCover_Forest,67472,SquareKilometer, +dcid:wikidataId/Q1164,1987,dcid:Area_LandCover_Forest,25160,SquareKilometer, +dcid:wikidataId/Q1164,1989,dcid:Area_LandCover_Forest,24832,SquareKilometer, +dcid:wikidataId/Q1164,1991,dcid:Area_LandCover_Forest,24751,SquareKilometer, +dcid:wikidataId/Q1164,1993,dcid:Area_LandCover_Forest,24508,SquareKilometer, +dcid:wikidataId/Q1164,1995,dcid:Area_LandCover_Forest,24061,SquareKilometer, +dcid:wikidataId/Q1164,1997,dcid:Area_LandCover_Forest,23824,SquareKilometer, +dcid:wikidataId/Q1164,1999,dcid:Area_LandCover_Forest,23688,SquareKilometer, +dcid:wikidataId/Q1164,2001,dcid:Area_LandCover_Forest,27714,SquareKilometer, +dcid:wikidataId/Q1164,2003,dcid:Area_LandCover_Forest,27826,SquareKilometer, +dcid:wikidataId/Q1164,2005,dcid:Area_LandCover_Forest,27758,SquareKilometer, +dcid:wikidataId/Q1165,1987,dcid:Area_LandCover_Forest,28482,SquareKilometer, +dcid:wikidataId/Q1165,1989,dcid:Area_LandCover_Forest,26668,SquareKilometer, +dcid:wikidataId/Q1165,1991,dcid:Area_LandCover_Forest,26668,SquareKilometer, +dcid:wikidataId/Q1165,1993,dcid:Area_LandCover_Forest,26587,SquareKilometer, +dcid:wikidataId/Q1165,1995,dcid:Area_LandCover_Forest,26561,SquareKilometer, +dcid:wikidataId/Q1165,1997,dcid:Area_LandCover_Forest,4832,SquareKilometer, +dcid:wikidataId/Q1165,1999,dcid:Area_LandCover_Forest,4830,SquareKilometer, +dcid:wikidataId/Q1165,2001,dcid:Area_LandCover_Forest,5720,SquareKilometer, +dcid:wikidataId/Q1165,2003,dcid:Area_LandCover_Forest,5558,SquareKilometer, +dcid:wikidataId/Q1165,2005,dcid:Area_LandCover_Forest,6807,SquareKilometer, +dcid:wikidataId/Q43433,1987,dcid:Area_LandCover_Forest,2,SquareKilometer, +dcid:wikidataId/Q43433,1989,dcid:Area_LandCover_Forest,5,SquareKilometer, +dcid:wikidataId/Q43433,1991,dcid:Area_LandCover_Forest,5,SquareKilometer, +dcid:wikidataId/Q43433,1993,dcid:Area_LandCover_Forest,5,SquareKilometer, +dcid:wikidataId/Q43433,1995,dcid:Area_LandCover_Forest,7,SquareKilometer, +dcid:wikidataId/Q43433,1997,dcid:Area_LandCover_Forest,7,SquareKilometer, +dcid:wikidataId/Q43433,1999,dcid:Area_LandCover_Forest,7,SquareKilometer, +dcid:wikidataId/Q43433,2001,dcid:Area_LandCover_Forest,9,SquareKilometer, +dcid:wikidataId/Q43433,2003,dcid:Area_LandCover_Forest,15,SquareKilometer, +dcid:wikidataId/Q43433,2005,dcid:Area_LandCover_Forest,17,SquareKilometer, +dcid:wikidataId/Q1168,1997,dcid:Area_LandCover_Forest,56435,SquareKilometer, +dcid:wikidataId/Q1168,1999,dcid:Area_LandCover_Forest,56693,SquareKilometer, +dcid:wikidataId/Q1168,2001,dcid:Area_LandCover_Forest,56448,SquareKilometer, +dcid:wikidataId/Q1168,2003,dcid:Area_LandCover_Forest,55998,SquareKilometer, +dcid:wikidataId/Q1168,2005,dcid:Area_LandCover_Forest,55929,SquareKilometer, +dcid:wikidataId/Q46107,1987,dcid:Area_LandCover_Forest,238,SquareKilometer, +dcid:wikidataId/Q46107,1989,dcid:Area_LandCover_Forest,206,SquareKilometer, +dcid:wikidataId/Q46107,1991,dcid:Area_LandCover_Forest,206,SquareKilometer, +dcid:wikidataId/Q46107,1993,dcid:Area_LandCover_Forest,206,SquareKilometer, +dcid:wikidataId/Q46107,1995,dcid:Area_LandCover_Forest,204,SquareKilometer, +dcid:wikidataId/Q46107,1997,dcid:Area_LandCover_Forest,204,SquareKilometer, +dcid:wikidataId/Q46107,1999,dcid:Area_LandCover_Forest,202,SquareKilometer, +dcid:wikidataId/Q46107,2001,dcid:Area_LandCover_Forest,219,SquareKilometer, +dcid:wikidataId/Q46107,2003,dcid:Area_LandCover_Forest,225,SquareKilometer, +dcid:wikidataId/Q46107,2005,dcid:Area_LandCover_Forest,216,SquareKilometer, +dcid:wikidataId/Q66710,1995,dcid:Area_LandCover_Forest,3,SquareKilometer, +dcid:wikidataId/Q66710,1997,dcid:Area_LandCover_Forest,3,SquareKilometer, +dcid:wikidataId/Q66710,2001,dcid:Area_LandCover_Forest,6,SquareKilometer, +dcid:wikidataId/Q66710,2003,dcid:Area_LandCover_Forest,8,SquareKilometer, +dcid:wikidataId/Q66710,2005,dcid:Area_LandCover_Forest,6,SquareKilometer, +dcid:wikidataId/Q1353,1987,dcid:Area_LandCover_Forest,15,SquareKilometer, +dcid:wikidataId/Q1353,1989,dcid:Area_LandCover_Forest,22,SquareKilometer, +dcid:wikidataId/Q1353,1991,dcid:Area_LandCover_Forest,22,SquareKilometer, +dcid:wikidataId/Q1353,1993,dcid:Area_LandCover_Forest,22,SquareKilometer, +dcid:wikidataId/Q1353,1995,dcid:Area_LandCover_Forest,26,SquareKilometer, +dcid:wikidataId/Q1353,1997,dcid:Area_LandCover_Forest,26,SquareKilometer, +dcid:wikidataId/Q1353,1999,dcid:Area_LandCover_Forest,88,SquareKilometer, +dcid:wikidataId/Q1353,2001,dcid:Area_LandCover_Forest,111,SquareKilometer, +dcid:wikidataId/Q1353,2003,dcid:Area_LandCover_Forest,170,SquareKilometer, +dcid:wikidataId/Q1353,2005,dcid:Area_LandCover_Forest,177,SquareKilometer, +dcid:wikidataId/Q1171,1987,dcid:Area_LandCover_Forest,1240,SquareKilometer, +dcid:wikidataId/Q1171,1989,dcid:Area_LandCover_Forest,1255,SquareKilometer, +dcid:wikidataId/Q1171,1991,dcid:Area_LandCover_Forest,1255,SquareKilometer, +dcid:wikidataId/Q1171,1993,dcid:Area_LandCover_Forest,1250,SquareKilometer, +dcid:wikidataId/Q1171,1995,dcid:Area_LandCover_Forest,1247,SquareKilometer, +dcid:wikidataId/Q1171,1997,dcid:Area_LandCover_Forest,1252,SquareKilometer, +dcid:wikidataId/Q1171,1999,dcid:Area_LandCover_Forest,1251,SquareKilometer, +dcid:wikidataId/Q1171,2001,dcid:Area_LandCover_Forest,2095,SquareKilometer, +dcid:wikidataId/Q1171,2003,dcid:Area_LandCover_Forest,2156,SquareKilometer, +dcid:wikidataId/Q1171,2005,dcid:Area_LandCover_Forest,2156,SquareKilometer, +dcid:wikidataId/Q1061,1987,dcid:Area_LandCover_Forest,11991,SquareKilometer, +dcid:wikidataId/Q1061,1989,dcid:Area_LandCover_Forest,11921,SquareKilometer, +dcid:wikidataId/Q1061,1991,dcid:Area_LandCover_Forest,11907,SquareKilometer, +dcid:wikidataId/Q1061,1993,dcid:Area_LandCover_Forest,12044,SquareKilometer, +dcid:wikidataId/Q1061,1995,dcid:Area_LandCover_Forest,12320,SquareKilometer, +dcid:wikidataId/Q1061,1997,dcid:Area_LandCover_Forest,12578,SquareKilometer, +dcid:wikidataId/Q1061,1999,dcid:Area_LandCover_Forest,12965,SquareKilometer, +dcid:wikidataId/Q1061,2001,dcid:Area_LandCover_Forest,15152,SquareKilometer, +dcid:wikidataId/Q1061,2003,dcid:Area_LandCover_Forest,14946,SquareKilometer, +dcid:wikidataId/Q1061,2005,dcid:Area_LandCover_Forest,14604,SquareKilometer, +dcid:wikidataId/Q1174,1987,dcid:Area_LandCover_Forest,513,SquareKilometer, +dcid:wikidataId/Q1174,1989,dcid:Area_LandCover_Forest,513,SquareKilometer, +dcid:wikidataId/Q1174,1991,dcid:Area_LandCover_Forest,513,SquareKilometer, +dcid:wikidataId/Q1174,1993,dcid:Area_LandCover_Forest,513,SquareKilometer, +dcid:wikidataId/Q1174,1995,dcid:Area_LandCover_Forest,603,SquareKilometer, +dcid:wikidataId/Q1174,1997,dcid:Area_LandCover_Forest,604,SquareKilometer, +dcid:wikidataId/Q1174,1999,dcid:Area_LandCover_Forest,964,SquareKilometer, +dcid:wikidataId/Q1174,2001,dcid:Area_LandCover_Forest,1754,SquareKilometer, +dcid:wikidataId/Q1174,2003,dcid:Area_LandCover_Forest,1517,SquareKilometer, +dcid:wikidataId/Q1174,2005,dcid:Area_LandCover_Forest,1604,SquareKilometer, +dcid:wikidataId/Q1177,1987,dcid:Area_LandCover_Forest,12480,SquareKilometer, +dcid:wikidataId/Q1177,1989,dcid:Area_LandCover_Forest,12480,SquareKilometer, +dcid:wikidataId/Q1177,1991,dcid:Area_LandCover_Forest,12480,SquareKilometer, +dcid:wikidataId/Q1177,1993,dcid:Area_LandCover_Forest,12502,SquareKilometer, +dcid:wikidataId/Q1177,1995,dcid:Area_LandCover_Forest,12501,SquareKilometer, +dcid:wikidataId/Q1177,1997,dcid:Area_LandCover_Forest,12521,SquareKilometer, +dcid:wikidataId/Q1177,1999,dcid:Area_LandCover_Forest,13082,SquareKilometer, +dcid:wikidataId/Q1177,2001,dcid:Area_LandCover_Forest,14360,SquareKilometer, +dcid:wikidataId/Q1177,2003,dcid:Area_LandCover_Forest,14353,SquareKilometer, +dcid:wikidataId/Q1177,2005,dcid:Area_LandCover_Forest,14666,SquareKilometer, +dcid:wikidataId/Q66278313,1987,dcid:Area_LandCover_Forest,20905,SquareKilometer, +dcid:wikidataId/Q66278313,1989,dcid:Area_LandCover_Forest,20499,SquareKilometer, +dcid:wikidataId/Q66278313,1991,dcid:Area_LandCover_Forest,20499,SquareKilometer, +dcid:wikidataId/Q66278313,1993,dcid:Area_LandCover_Forest,20443,SquareKilometer, +dcid:wikidataId/Q66278313,1995,dcid:Area_LandCover_Forest,20433,SquareKilometer, +dcid:wikidataId/Q66278313,1997,dcid:Area_LandCover_Forest,20440,SquareKilometer, +dcid:wikidataId/Q66278313,1999,dcid:Area_LandCover_Forest,20441,SquareKilometer, +dcid:wikidataId/Q66278313,2001,dcid:Area_LandCover_Forest,21237,SquareKilometer, +dcid:wikidataId/Q66278313,2003,dcid:Area_LandCover_Forest,21267,SquareKilometer, +dcid:wikidataId/Q66278313,2005,dcid:Area_LandCover_Forest,22689,SquareKilometer, +dcid:wikidataId/Q1184,1997,dcid:Area_LandCover_Forest,21692,SquareKilometer, +dcid:wikidataId/Q1184,1999,dcid:Area_LandCover_Forest,21644,SquareKilometer, +dcid:wikidataId/Q1184,2001,dcid:Area_LandCover_Forest,22637,SquareKilometer, +dcid:wikidataId/Q1184,2003,dcid:Area_LandCover_Forest,22716,SquareKilometer, +dcid:wikidataId/Q1184,2005,dcid:Area_LandCover_Forest,22722,SquareKilometer, +dcid:wikidataId/Q1185,1987,dcid:Area_LandCover_Forest,32268,SquareKilometer, +dcid:wikidataId/Q1185,1989,dcid:Area_LandCover_Forest,32104,SquareKilometer, +dcid:wikidataId/Q1185,1991,dcid:Area_LandCover_Forest,32199,SquareKilometer, +dcid:wikidataId/Q1185,1993,dcid:Area_LandCover_Forest,32343,SquareKilometer, +dcid:wikidataId/Q1185,1995,dcid:Area_LandCover_Forest,32382,SquareKilometer, +dcid:wikidataId/Q1185,1997,dcid:Area_LandCover_Forest,32403,SquareKilometer, +dcid:wikidataId/Q1185,1999,dcid:Area_LandCover_Forest,32467,SquareKilometer, +dcid:wikidataId/Q1185,2001,dcid:Area_LandCover_Forest,36991,SquareKilometer, +dcid:wikidataId/Q1185,2003,dcid:Area_LandCover_Forest,36449,SquareKilometer, +dcid:wikidataId/Q1185,2005,dcid:Area_LandCover_Forest,36200,SquareKilometer, +dcid:wikidataId/Q1186,1987,dcid:Area_LandCover_Forest,10292,SquareKilometer, +dcid:wikidataId/Q1186,1989,dcid:Area_LandCover_Forest,10292,SquareKilometer, +dcid:wikidataId/Q1186,1991,dcid:Area_LandCover_Forest,10292,SquareKilometer, +dcid:wikidataId/Q1186,1993,dcid:Area_LandCover_Forest,10336,SquareKilometer, +dcid:wikidataId/Q1186,1995,dcid:Area_LandCover_Forest,10336,SquareKilometer, +dcid:wikidataId/Q1186,1997,dcid:Area_LandCover_Forest,10334,SquareKilometer, +dcid:wikidataId/Q1186,1999,dcid:Area_LandCover_Forest,10323,SquareKilometer, +dcid:wikidataId/Q1186,2001,dcid:Area_LandCover_Forest,15560,SquareKilometer, +dcid:wikidataId/Q1186,2003,dcid:Area_LandCover_Forest,15577,SquareKilometer, +dcid:wikidataId/Q1186,2005,dcid:Area_LandCover_Forest,17284,SquareKilometer, +dcid:wikidataId/Q26927,2001,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q26927,2003,dcid:Area_LandCover_Forest,23,SquareKilometer, +dcid:wikidataId/Q26927,2005,dcid:Area_LandCover_Forest,26,SquareKilometer, +dcid:wikidataId/Q1188,1987,dcid:Area_LandCover_Forest,130099,SquareKilometer, +dcid:wikidataId/Q1188,1989,dcid:Area_LandCover_Forest,135541,SquareKilometer, +dcid:wikidataId/Q1188,1991,dcid:Area_LandCover_Forest,135541,SquareKilometer, +dcid:wikidataId/Q1188,1993,dcid:Area_LandCover_Forest,135396,SquareKilometer, +dcid:wikidataId/Q1188,1995,dcid:Area_LandCover_Forest,135164,SquareKilometer, +dcid:wikidataId/Q1188,1997,dcid:Area_LandCover_Forest,74760,SquareKilometer, +dcid:wikidataId/Q1188,1999,dcid:Area_LandCover_Forest,75137,SquareKilometer, +dcid:wikidataId/Q1188,2001,dcid:Area_LandCover_Forest,77265,SquareKilometer, +dcid:wikidataId/Q1188,2003,dcid:Area_LandCover_Forest,76429,SquareKilometer, +dcid:wikidataId/Q1188,2005,dcid:Area_LandCover_Forest,77739,SquareKilometer, +dcid:wikidataId/Q1191,1987,dcid:Area_LandCover_Forest,45616,SquareKilometer, +dcid:wikidataId/Q1191,1989,dcid:Area_LandCover_Forest,44044,SquareKilometer, +dcid:wikidataId/Q1191,1991,dcid:Area_LandCover_Forest,44044,SquareKilometer, +dcid:wikidataId/Q1191,1993,dcid:Area_LandCover_Forest,43859,SquareKilometer, +dcid:wikidataId/Q1191,1995,dcid:Area_LandCover_Forest,43843,SquareKilometer, +dcid:wikidataId/Q1191,1997,dcid:Area_LandCover_Forest,46143,SquareKilometer, +dcid:wikidataId/Q1191,1999,dcid:Area_LandCover_Forest,46672,SquareKilometer, +dcid:wikidataId/Q1191,2001,dcid:Area_LandCover_Forest,47482,SquareKilometer, +dcid:wikidataId/Q1191,2003,dcid:Area_LandCover_Forest,46865,SquareKilometer, +dcid:wikidataId/Q1191,2005,dcid:Area_LandCover_Forest,50661,SquareKilometer, +dcid:wikidataId/Q1193,1987,dcid:Area_LandCover_Forest,17475,SquareKilometer, +dcid:wikidataId/Q1193,1989,dcid:Area_LandCover_Forest,17685,SquareKilometer, +dcid:wikidataId/Q1193,1991,dcid:Area_LandCover_Forest,17685,SquareKilometer, +dcid:wikidataId/Q1193,1993,dcid:Area_LandCover_Forest,17621,SquareKilometer, +dcid:wikidataId/Q1193,1995,dcid:Area_LandCover_Forest,17558,SquareKilometer, +dcid:wikidataId/Q1193,1997,dcid:Area_LandCover_Forest,17418,SquareKilometer, +dcid:wikidataId/Q1193,1999,dcid:Area_LandCover_Forest,17384,SquareKilometer, +dcid:wikidataId/Q1193,2001,dcid:Area_LandCover_Forest,16926,SquareKilometer, +dcid:wikidataId/Q1193,2003,dcid:Area_LandCover_Forest,17219,SquareKilometer, +dcid:wikidataId/Q1193,2005,dcid:Area_LandCover_Forest,16952,SquareKilometer, +dcid:wikidataId/Q1195,1987,dcid:Area_LandCover_Forest,16466,SquareKilometer, +dcid:wikidataId/Q1195,1989,dcid:Area_LandCover_Forest,15645,SquareKilometer, +dcid:wikidataId/Q1195,1991,dcid:Area_LandCover_Forest,15875,SquareKilometer, +dcid:wikidataId/Q1195,1993,dcid:Area_LandCover_Forest,15769,SquareKilometer, +dcid:wikidataId/Q1195,1995,dcid:Area_LandCover_Forest,15714,SquareKilometer, +dcid:wikidataId/Q1195,1997,dcid:Area_LandCover_Forest,15657,SquareKilometer, +dcid:wikidataId/Q1195,1999,dcid:Area_LandCover_Forest,15633,SquareKilometer, +dcid:wikidataId/Q1195,2001,dcid:Area_LandCover_Forest,15584,SquareKilometer, +dcid:wikidataId/Q1195,2003,dcid:Area_LandCover_Forest,16839,SquareKilometer, +dcid:wikidataId/Q1195,2005,dcid:Area_LandCover_Forest,17205,SquareKilometer, +dcid:wikidataId/Q1502,1987,dcid:Area_LandCover_Forest,19084,SquareKilometer, +dcid:wikidataId/Q1502,1989,dcid:Area_LandCover_Forest,18170,SquareKilometer, +dcid:wikidataId/Q1502,1991,dcid:Area_LandCover_Forest,18853,SquareKilometer, +dcid:wikidataId/Q1502,1993,dcid:Area_LandCover_Forest,18697,SquareKilometer, +dcid:wikidataId/Q1502,1995,dcid:Area_LandCover_Forest,18576,SquareKilometer, +dcid:wikidataId/Q1502,1997,dcid:Area_LandCover_Forest,18775,SquareKilometer, +dcid:wikidataId/Q1502,1999,dcid:Area_LandCover_Forest,18338,SquareKilometer, +dcid:wikidataId/Q1502,2001,dcid:Area_LandCover_Forest,17494,SquareKilometer, +dcid:wikidataId/Q1502,2003,dcid:Area_LandCover_Forest,18430,SquareKilometer, +dcid:wikidataId/Q1502,2005,dcid:Area_LandCover_Forest,18600,SquareKilometer, +dcid:wikidataId/Q1599,1987,dcid:Area_LandCover_Forest,14394,SquareKilometer, +dcid:wikidataId/Q1599,1989,dcid:Area_LandCover_Forest,14399,SquareKilometer, +dcid:wikidataId/Q1599,1991,dcid:Area_LandCover_Forest,14321,SquareKilometer, +dcid:wikidataId/Q1599,1993,dcid:Area_LandCover_Forest,14348,SquareKilometer, +dcid:wikidataId/Q1599,1995,dcid:Area_LandCover_Forest,14291,SquareKilometer, +dcid:wikidataId/Q1599,1997,dcid:Area_LandCover_Forest,14221,SquareKilometer, +dcid:wikidataId/Q1599,1999,dcid:Area_LandCover_Forest,14164,SquareKilometer, +dcid:wikidataId/Q1599,2001,dcid:Area_LandCover_Forest,13345,SquareKilometer, +dcid:wikidataId/Q1599,2003,dcid:Area_LandCover_Forest,13609,SquareKilometer, +dcid:wikidataId/Q1599,2005,dcid:Area_LandCover_Forest,13665,SquareKilometer, +dcid:wikidataId/Q22048,1987,dcid:Area_LandCover_Forest,53253,SquareKilometer, +dcid:wikidataId/Q22048,1989,dcid:Area_LandCover_Forest,47227,SquareKilometer, +dcid:wikidataId/Q22048,1991,dcid:Area_LandCover_Forest,47205,SquareKilometer, +dcid:wikidataId/Q22048,1993,dcid:Area_LandCover_Forest,47145,SquareKilometer, +dcid:wikidataId/Q22048,1995,dcid:Area_LandCover_Forest,47107,SquareKilometer, +dcid:wikidataId/Q22048,1997,dcid:Area_LandCover_Forest,46941,SquareKilometer, +dcid:wikidataId/Q22048,1999,dcid:Area_LandCover_Forest,47033,SquareKilometer, +dcid:wikidataId/Q22048,2001,dcid:Area_LandCover_Forest,48838,SquareKilometer, +dcid:wikidataId/Q22048,2003,dcid:Area_LandCover_Forest,48366,SquareKilometer, +dcid:wikidataId/Q22048,2005,dcid:Area_LandCover_Forest,48755,SquareKilometer, +dcid:wikidataId/Q66743,2001,dcid:Area_LandCover_Forest,36,SquareKilometer, +dcid:wikidataId/Q66743,2003,dcid:Area_LandCover_Forest,40,SquareKilometer, +dcid:wikidataId/Q66743,2005,dcid:Area_LandCover_Forest,42,SquareKilometer, +dcid:wikidataId/Q22424,1987,dcid:Area_LandCover_Forest,943,SquareKilometer, +dcid:wikidataId/Q22424,1989,dcid:Area_LandCover_Forest,1338,SquareKilometer, +dcid:wikidataId/Q22424,1991,dcid:Area_LandCover_Forest,1343,SquareKilometer, +dcid:wikidataId/Q22424,1993,dcid:Area_LandCover_Forest,1343,SquareKilometer, +dcid:wikidataId/Q22424,1995,dcid:Area_LandCover_Forest,1342,SquareKilometer, +dcid:wikidataId/Q22424,1997,dcid:Area_LandCover_Forest,1387,SquareKilometer, +dcid:wikidataId/Q22424,1999,dcid:Area_LandCover_Forest,1412,SquareKilometer, +dcid:wikidataId/Q22424,2001,dcid:Area_LandCover_Forest,2432,SquareKilometer, +dcid:wikidataId/Q22424,2003,dcid:Area_LandCover_Forest,1580,SquareKilometer, +dcid:wikidataId/Q22424,2005,dcid:Area_LandCover_Forest,1660,SquareKilometer, +dcid:wikidataId/Q1437,1987,dcid:Area_LandCover_Forest,12758,SquareKilometer, +dcid:wikidataId/Q1437,1989,dcid:Area_LandCover_Forest,12884,SquareKilometer, +dcid:wikidataId/Q1437,1991,dcid:Area_LandCover_Forest,12889,SquareKilometer, +dcid:wikidataId/Q1437,1993,dcid:Area_LandCover_Forest,13099,SquareKilometer, +dcid:wikidataId/Q1437,1995,dcid:Area_LandCover_Forest,13280,SquareKilometer, +dcid:wikidataId/Q1437,1997,dcid:Area_LandCover_Forest,13353,SquareKilometer, +dcid:wikidataId/Q1437,1999,dcid:Area_LandCover_Forest,13871,SquareKilometer, +dcid:wikidataId/Q1437,2001,dcid:Area_LandCover_Forest,16367,SquareKilometer, +dcid:wikidataId/Q1437,2003,dcid:Area_LandCover_Forest,15826,SquareKilometer, +dcid:wikidataId/Q1437,2005,dcid:Area_LandCover_Forest,16012,SquareKilometer, +dcid:wikidataId/Q1505,1987,dcid:Area_LandCover_Forest,2756,SquareKilometer, +dcid:wikidataId/Q1505,1989,dcid:Area_LandCover_Forest,3041,SquareKilometer, +dcid:wikidataId/Q1505,1991,dcid:Area_LandCover_Forest,3041,SquareKilometer, +dcid:wikidataId/Q1505,1993,dcid:Area_LandCover_Forest,3119,SquareKilometer, +dcid:wikidataId/Q1505,1995,dcid:Area_LandCover_Forest,3127,SquareKilometer, +dcid:wikidataId/Q1505,1997,dcid:Area_LandCover_Forest,3129,SquareKilometer, +dcid:wikidataId/Q1505,1999,dcid:Area_LandCover_Forest,3118,SquareKilometer, +dcid:wikidataId/Q1505,2001,dcid:Area_LandCover_Forest,3193,SquareKilometer, +dcid:wikidataId/Q1505,2003,dcid:Area_LandCover_Forest,3262,SquareKilometer, +dcid:wikidataId/Q1505,2005,dcid:Area_LandCover_Forest,3357,SquareKilometer, +dcid:wikidataId/Q1445,1987,dcid:Area_LandCover_Forest,17472,SquareKilometer, +dcid:wikidataId/Q1445,1989,dcid:Area_LandCover_Forest,16992,SquareKilometer, +dcid:wikidataId/Q1445,1991,dcid:Area_LandCover_Forest,16992,SquareKilometer, +dcid:wikidataId/Q1445,1993,dcid:Area_LandCover_Forest,17005,SquareKilometer, +dcid:wikidataId/Q1445,1995,dcid:Area_LandCover_Forest,17045,SquareKilometer, +dcid:wikidataId/Q1445,1997,dcid:Area_LandCover_Forest,17064,SquareKilometer, +dcid:wikidataId/Q1445,1999,dcid:Area_LandCover_Forest,17078,SquareKilometer, +dcid:wikidataId/Q1445,2001,dcid:Area_LandCover_Forest,21482,SquareKilometer, +dcid:wikidataId/Q1445,2003,dcid:Area_LandCover_Forest,22643,SquareKilometer, +dcid:wikidataId/Q1445,2005,dcid:Area_LandCover_Forest,23314,SquareKilometer, +dcid:wikidataId/Q1363,1987,dcid:Area_LandCover_Forest,5953,SquareKilometer, +dcid:wikidataId/Q1363,1989,dcid:Area_LandCover_Forest,5535,SquareKilometer, +dcid:wikidataId/Q1363,1991,dcid:Area_LandCover_Forest,5535,SquareKilometer, +dcid:wikidataId/Q1363,1993,dcid:Area_LandCover_Forest,5538,SquareKilometer, +dcid:wikidataId/Q1363,1995,dcid:Area_LandCover_Forest,5538,SquareKilometer, +dcid:wikidataId/Q1363,1997,dcid:Area_LandCover_Forest,5546,SquareKilometer, +dcid:wikidataId/Q1363,1999,dcid:Area_LandCover_Forest,5745,SquareKilometer, +dcid:wikidataId/Q1363,2001,dcid:Area_LandCover_Forest,7065,SquareKilometer, +dcid:wikidataId/Q1363,2003,dcid:Area_LandCover_Forest,8093,SquareKilometer, +dcid:wikidataId/Q1363,2005,dcid:Area_LandCover_Forest,8173,SquareKilometer, +dcid:wikidataId/Q1498,1987,dcid:Area_LandCover_Forest,31226,SquareKilometer, +dcid:wikidataId/Q1498,1989,dcid:Area_LandCover_Forest,33627,SquareKilometer, +dcid:wikidataId/Q1498,1991,dcid:Area_LandCover_Forest,33609,SquareKilometer, +dcid:wikidataId/Q1498,1993,dcid:Area_LandCover_Forest,33961,SquareKilometer, +dcid:wikidataId/Q1498,1995,dcid:Area_LandCover_Forest,33986,SquareKilometer, +dcid:wikidataId/Q1498,1997,dcid:Area_LandCover_Forest,10751,SquareKilometer, +dcid:wikidataId/Q1498,1999,dcid:Area_LandCover_Forest,10756,SquareKilometer, +dcid:wikidataId/Q1498,2001,dcid:Area_LandCover_Forest,13746,SquareKilometer, +dcid:wikidataId/Q1498,2003,dcid:Area_LandCover_Forest,14118,SquareKilometer, +dcid:wikidataId/Q1498,2005,dcid:Area_LandCover_Forest,14346,SquareKilometer, +dcid:wikidataId/Q1499,1997,dcid:Area_LandCover_Forest,23243,SquareKilometer, +dcid:wikidataId/Q1499,1999,dcid:Area_LandCover_Forest,23260,SquareKilometer, +dcid:wikidataId/Q1499,2001,dcid:Area_LandCover_Forest,23938,SquareKilometer, +dcid:wikidataId/Q1499,2003,dcid:Area_LandCover_Forest,24465,SquareKilometer, +dcid:wikidataId/Q1499,2005,dcid:Area_LandCover_Forest,24493,SquareKilometer, +dcid:wikidataId/Q1356,1987,dcid:Area_LandCover_Forest,8432,SquareKilometer, +dcid:wikidataId/Q1356,1989,dcid:Area_LandCover_Forest,8015,SquareKilometer, +dcid:wikidataId/Q1356,1991,dcid:Area_LandCover_Forest,8015,SquareKilometer, +dcid:wikidataId/Q1356,1993,dcid:Area_LandCover_Forest,8186,SquareKilometer, +dcid:wikidataId/Q1356,1995,dcid:Area_LandCover_Forest,8276,SquareKilometer, +dcid:wikidataId/Q1356,1997,dcid:Area_LandCover_Forest,8349,SquareKilometer, +dcid:wikidataId/Q1356,1999,dcid:Area_LandCover_Forest,8362,SquareKilometer, +dcid:wikidataId/Q1356,2001,dcid:Area_LandCover_Forest,10693,SquareKilometer, +dcid:wikidataId/Q1356,2003,dcid:Area_LandCover_Forest,12343,SquareKilometer, +dcid:wikidataId/Q1356,2005,dcid:Area_LandCover_Forest,12970,SquareKilometer, +dcid:country/IND,1987,dcid:Area_LandCover_Forest,640819,SquareKilometer, +dcid:country/IND,1989,dcid:Area_LandCover_Forest,638804,SquareKilometer, +dcid:country/IND,1991,dcid:Area_LandCover_Forest,639364,SquareKilometer, +dcid:country/IND,1993,dcid:Area_LandCover_Forest,639386,SquareKilometer, +dcid:country/IND,1995,dcid:Area_LandCover_Forest,638879,SquareKilometer, +dcid:country/IND,1997,dcid:Area_LandCover_Forest,633397,SquareKilometer, +dcid:country/IND,1999,dcid:Area_LandCover_Forest,637293,SquareKilometer, +dcid:country/IND,2001,dcid:Area_LandCover_Forest,675538,SquareKilometer, +dcid:country/IND,2003,dcid:Area_LandCover_Forest,678333,SquareKilometer, +dcid:country/IND,2005,dcid:Area_LandCover_Forest,690171,SquareKilometer, +dcid:wikidataId/Q40888,2007,dcid:Area_LandCover_Forest,6662,SquareKilometer, +dcid:wikidataId/Q40888,2009,dcid:Area_LandCover_Forest,6662,SquareKilometer, +dcid:wikidataId/Q40888,2011,dcid:Area_LandCover_Forest,6724,SquareKilometer, +dcid:wikidataId/Q40888,2013,dcid:Area_LandCover_Forest,6711,SquareKilometer, +dcid:wikidataId/Q40888,2015,dcid:Area_LandCover_Forest,6751,SquareKilometer, +dcid:wikidataId/Q40888,2017,dcid:Area_LandCover_Forest,6742,SquareKilometer, +dcid:wikidataId/Q40888,2019,dcid:Area_LandCover_Forest,6743,SquareKilometer, +dcid:wikidataId/Q40888,2021,dcid:Area_LandCover_Forest,6737,SquareKilometer, +dcid:wikidataId/Q40888,2023,dcid:Area_LandCover_Forest,6733,SquareKilometer, +dcid:wikidataId/Q1159,2007,dcid:Area_LandCover_Forest,45102,SquareKilometer, +dcid:wikidataId/Q1159,2009,dcid:Area_LandCover_Forest,46670,SquareKilometer, +dcid:wikidataId/Q1159,2011,dcid:Area_LandCover_Forest,46389,SquareKilometer, +dcid:wikidataId/Q1159,2013,dcid:Area_LandCover_Forest,26044,SquareKilometer, +dcid:wikidataId/Q1159,2015,dcid:Area_LandCover_Forest,26006,SquareKilometer, +dcid:wikidataId/Q1159,2017,dcid:Area_LandCover_Forest,28147,SquareKilometer, +dcid:wikidataId/Q1159,2019,dcid:Area_LandCover_Forest,29137,SquareKilometer, +dcid:wikidataId/Q1159,2021,dcid:Area_LandCover_Forest,30224,SquareKilometer, +dcid:wikidataId/Q1159,2023,dcid:Area_LandCover_Forest,30085,SquareKilometer, +dcid:wikidataId/Q1162,2007,dcid:Area_LandCover_Forest,67353,SquareKilometer, +dcid:wikidataId/Q1162,2009,dcid:Area_LandCover_Forest,67484,SquareKilometer, +dcid:wikidataId/Q1162,2011,dcid:Area_LandCover_Forest,67410,SquareKilometer, +dcid:wikidataId/Q1162,2013,dcid:Area_LandCover_Forest,66966,SquareKilometer, +dcid:wikidataId/Q1162,2015,dcid:Area_LandCover_Forest,67154,SquareKilometer, +dcid:wikidataId/Q1162,2017,dcid:Area_LandCover_Forest,66964,SquareKilometer, +dcid:wikidataId/Q1162,2019,dcid:Area_LandCover_Forest,66688,SquareKilometer, +dcid:wikidataId/Q1162,2021,dcid:Area_LandCover_Forest,65973,SquareKilometer, +dcid:wikidataId/Q1162,2023,dcid:Area_LandCover_Forest,65882,SquareKilometer, +dcid:wikidataId/Q1164,2007,dcid:Area_LandCover_Forest,27692,SquareKilometer, +dcid:wikidataId/Q1164,2009,dcid:Area_LandCover_Forest,27692,SquareKilometer, +dcid:wikidataId/Q1164,2011,dcid:Area_LandCover_Forest,27673,SquareKilometer, +dcid:wikidataId/Q1164,2013,dcid:Area_LandCover_Forest,27565,SquareKilometer, +dcid:wikidataId/Q1164,2015,dcid:Area_LandCover_Forest,27538,SquareKilometer, +dcid:wikidataId/Q1164,2017,dcid:Area_LandCover_Forest,28105,SquareKilometer, +dcid:wikidataId/Q1164,2019,dcid:Area_LandCover_Forest,28327,SquareKilometer, +dcid:wikidataId/Q1164,2021,dcid:Area_LandCover_Forest,28325,SquareKilometer, +dcid:wikidataId/Q1164,2023,dcid:Area_LandCover_Forest,28314,SquareKilometer, +dcid:wikidataId/Q1165,2007,dcid:Area_LandCover_Forest,6804,SquareKilometer, +dcid:wikidataId/Q1165,2009,dcid:Area_LandCover_Forest,6804,SquareKilometer, +dcid:wikidataId/Q1165,2011,dcid:Area_LandCover_Forest,6845,SquareKilometer, +dcid:wikidataId/Q1165,2013,dcid:Area_LandCover_Forest,7284,SquareKilometer, +dcid:wikidataId/Q1165,2015,dcid:Area_LandCover_Forest,7254,SquareKilometer, +dcid:wikidataId/Q1165,2017,dcid:Area_LandCover_Forest,7299,SquareKilometer, +dcid:wikidataId/Q1165,2019,dcid:Area_LandCover_Forest,7306,SquareKilometer, +dcid:wikidataId/Q1165,2021,dcid:Area_LandCover_Forest,7403,SquareKilometer, +dcid:wikidataId/Q1165,2023,dcid:Area_LandCover_Forest,7532,SquareKilometer, +dcid:wikidataId/Q43433,2007,dcid:Area_LandCover_Forest,17,SquareKilometer, +dcid:wikidataId/Q43433,2009,dcid:Area_LandCover_Forest,17,SquareKilometer, +dcid:wikidataId/Q43433,2011,dcid:Area_LandCover_Forest,17,SquareKilometer, +dcid:wikidataId/Q43433,2013,dcid:Area_LandCover_Forest,17,SquareKilometer, +dcid:wikidataId/Q43433,2015,dcid:Area_LandCover_Forest,22,SquareKilometer, +dcid:wikidataId/Q43433,2017,dcid:Area_LandCover_Forest,22,SquareKilometer, +dcid:wikidataId/Q43433,2019,dcid:Area_LandCover_Forest,22,SquareKilometer, +dcid:wikidataId/Q43433,2021,dcid:Area_LandCover_Forest,24,SquareKilometer, +dcid:wikidataId/Q43433,2023,dcid:Area_LandCover_Forest,25,SquareKilometer, +dcid:wikidataId/Q1168,2007,dcid:Area_LandCover_Forest,55870,SquareKilometer, +dcid:wikidataId/Q1168,2009,dcid:Area_LandCover_Forest,55678,SquareKilometer, +dcid:wikidataId/Q1168,2011,dcid:Area_LandCover_Forest,55674,SquareKilometer, +dcid:wikidataId/Q1168,2013,dcid:Area_LandCover_Forest,55607,SquareKilometer, +dcid:wikidataId/Q1168,2015,dcid:Area_LandCover_Forest,55559,SquareKilometer, +dcid:wikidataId/Q1168,2017,dcid:Area_LandCover_Forest,55547,SquareKilometer, +dcid:wikidataId/Q1168,2019,dcid:Area_LandCover_Forest,55611,SquareKilometer, +dcid:wikidataId/Q1168,2021,dcid:Area_LandCover_Forest,55831,SquareKilometer, +dcid:wikidataId/Q1168,2023,dcid:Area_LandCover_Forest,55812,SquareKilometer, +dcid:wikidataId/Q46107,2007,dcid:Area_LandCover_Forest,211,SquareKilometer, +dcid:wikidataId/Q46107,2009,dcid:Area_LandCover_Forest,211,SquareKilometer, +dcid:wikidataId/Q46107,2011,dcid:Area_LandCover_Forest,211,SquareKilometer, +dcid:wikidataId/Q46107,2013,dcid:Area_LandCover_Forest,220,SquareKilometer, +dcid:wikidataId/Q46107,2015,dcid:Area_LandCover_Forest,206,SquareKilometer, +dcid:wikidataId/Q46107,2017,dcid:Area_LandCover_Forest,207,SquareKilometer, +dcid:wikidataId/Q46107,2019,dcid:Area_LandCover_Forest,207,SquareKilometer, +dcid:wikidataId/Q46107,2021,dcid:Area_LandCover_Forest,226,SquareKilometer, +dcid:wikidataId/Q46107,2023,dcid:Area_LandCover_Forest,226,SquareKilometer, +dcid:wikidataId/Q66710,2007,dcid:Area_LandCover_Forest,6,SquareKilometer, +dcid:wikidataId/Q66710,2009,dcid:Area_LandCover_Forest,6,SquareKilometer, +dcid:wikidataId/Q66710,2011,dcid:Area_LandCover_Forest,6,SquareKilometer, +dcid:wikidataId/Q66710,2015,dcid:Area_LandCover_Forest,20,SquareKilometer, +dcid:wikidataId/Q66710,2017,dcid:Area_LandCover_Forest,20,SquareKilometer, +dcid:wikidataId/Q66710,2019,dcid:Area_LandCover_Forest,20,SquareKilometer, +dcid:wikidataId/Q1353,2007,dcid:Area_LandCover_Forest,177,SquareKilometer, +dcid:wikidataId/Q1353,2009,dcid:Area_LandCover_Forest,177,SquareKilometer, +dcid:wikidataId/Q1353,2011,dcid:Area_LandCover_Forest,176,SquareKilometer, +dcid:wikidataId/Q1353,2013,dcid:Area_LandCover_Forest,180,SquareKilometer, +dcid:wikidataId/Q1353,2015,dcid:Area_LandCover_Forest,189,SquareKilometer, +dcid:wikidataId/Q1353,2017,dcid:Area_LandCover_Forest,192,SquareKilometer, +dcid:wikidataId/Q1353,2019,dcid:Area_LandCover_Forest,195,SquareKilometer, +dcid:wikidataId/Q1353,2021,dcid:Area_LandCover_Forest,195,SquareKilometer, +dcid:wikidataId/Q1353,2023,dcid:Area_LandCover_Forest,195,SquareKilometer, +dcid:wikidataId/Q1171,2007,dcid:Area_LandCover_Forest,2151,SquareKilometer, +dcid:wikidataId/Q1171,2009,dcid:Area_LandCover_Forest,2212,SquareKilometer, +dcid:wikidataId/Q1171,2011,dcid:Area_LandCover_Forest,2219,SquareKilometer, +dcid:wikidataId/Q1171,2013,dcid:Area_LandCover_Forest,2219,SquareKilometer, +dcid:wikidataId/Q1171,2015,dcid:Area_LandCover_Forest,2210,SquareKilometer, +dcid:wikidataId/Q1171,2017,dcid:Area_LandCover_Forest,2229,SquareKilometer, +dcid:wikidataId/Q1171,2019,dcid:Area_LandCover_Forest,2237,SquareKilometer, +dcid:wikidataId/Q1171,2021,dcid:Area_LandCover_Forest,2267,SquareKilometer, +dcid:wikidataId/Q1171,2023,dcid:Area_LandCover_Forest,2266,SquareKilometer, +dcid:wikidataId/Q1061,2007,dcid:Area_LandCover_Forest,14620,SquareKilometer, +dcid:wikidataId/Q1061,2009,dcid:Area_LandCover_Forest,14620,SquareKilometer, +dcid:wikidataId/Q1061,2011,dcid:Area_LandCover_Forest,14619,SquareKilometer, +dcid:wikidataId/Q1061,2013,dcid:Area_LandCover_Forest,14508,SquareKilometer, +dcid:wikidataId/Q1061,2015,dcid:Area_LandCover_Forest,14710,SquareKilometer, +dcid:wikidataId/Q1061,2017,dcid:Area_LandCover_Forest,14757,SquareKilometer, +dcid:wikidataId/Q1061,2019,dcid:Area_LandCover_Forest,14857,SquareKilometer, +dcid:wikidataId/Q1061,2021,dcid:Area_LandCover_Forest,14837,SquareKilometer, +dcid:wikidataId/Q1061,2023,dcid:Area_LandCover_Forest,15017,SquareKilometer, +dcid:wikidataId/Q1174,2007,dcid:Area_LandCover_Forest,1594,SquareKilometer, +dcid:wikidataId/Q1174,2009,dcid:Area_LandCover_Forest,1594,SquareKilometer, +dcid:wikidataId/Q1174,2011,dcid:Area_LandCover_Forest,1608,SquareKilometer, +dcid:wikidataId/Q1174,2013,dcid:Area_LandCover_Forest,1583,SquareKilometer, +dcid:wikidataId/Q1174,2015,dcid:Area_LandCover_Forest,1580,SquareKilometer, +dcid:wikidataId/Q1174,2017,dcid:Area_LandCover_Forest,1588,SquareKilometer, +dcid:wikidataId/Q1174,2019,dcid:Area_LandCover_Forest,1602,SquareKilometer, +dcid:wikidataId/Q1174,2021,dcid:Area_LandCover_Forest,1628,SquareKilometer, +dcid:wikidataId/Q1174,2023,dcid:Area_LandCover_Forest,1614,SquareKilometer, +dcid:wikidataId/Q1177,2007,dcid:Area_LandCover_Forest,14668,SquareKilometer, +dcid:wikidataId/Q1177,2009,dcid:Area_LandCover_Forest,14668,SquareKilometer, +dcid:wikidataId/Q1177,2011,dcid:Area_LandCover_Forest,14679,SquareKilometer, +dcid:wikidataId/Q1177,2013,dcid:Area_LandCover_Forest,14683,SquareKilometer, +dcid:wikidataId/Q1177,2015,dcid:Area_LandCover_Forest,14707,SquareKilometer, +dcid:wikidataId/Q1177,2017,dcid:Area_LandCover_Forest,15100,SquareKilometer, +dcid:wikidataId/Q1177,2019,dcid:Area_LandCover_Forest,15434,SquareKilometer, +dcid:wikidataId/Q1177,2021,dcid:Area_LandCover_Forest,15526,SquareKilometer, +dcid:wikidataId/Q1177,2023,dcid:Area_LandCover_Forest,15580,SquareKilometer, +dcid:wikidataId/Q66278313,2007,dcid:Area_LandCover_Forest,22686,SquareKilometer, +dcid:wikidataId/Q66278313,2009,dcid:Area_LandCover_Forest,22537,SquareKilometer, +dcid:wikidataId/Q66278313,2011,dcid:Area_LandCover_Forest,22539,SquareKilometer, +dcid:wikidataId/Q66278313,2013,dcid:Area_LandCover_Forest,20948,SquareKilometer, +dcid:wikidataId/Q66278313,2015,dcid:Area_LandCover_Forest,22988,SquareKilometer, +dcid:wikidataId/Q66278313,2017,dcid:Area_LandCover_Forest,23241,SquareKilometer, +dcid:wikidataId/Q66278313,2019,dcid:Area_LandCover_Forest,23612,SquareKilometer, +dcid:wikidataId/Q66278313,2021,dcid:Area_LandCover_Forest,21263,SquareKilometer, +dcid:wikidataId/Q66278313,2023,dcid:Area_LandCover_Forest,21346,SquareKilometer, +dcid:wikidataId/Q1184,2007,dcid:Area_LandCover_Forest,22894,SquareKilometer, +dcid:wikidataId/Q1184,2009,dcid:Area_LandCover_Forest,22894,SquareKilometer, +dcid:wikidataId/Q1184,2011,dcid:Area_LandCover_Forest,22977,SquareKilometer, +dcid:wikidataId/Q1184,2013,dcid:Area_LandCover_Forest,23815,SquareKilometer, +dcid:wikidataId/Q1184,2015,dcid:Area_LandCover_Forest,23524,SquareKilometer, +dcid:wikidataId/Q1184,2017,dcid:Area_LandCover_Forest,23553,SquareKilometer, +dcid:wikidataId/Q1184,2019,dcid:Area_LandCover_Forest,23611,SquareKilometer, +dcid:wikidataId/Q1184,2021,dcid:Area_LandCover_Forest,23707,SquareKilometer, +dcid:wikidataId/Q1184,2023,dcid:Area_LandCover_Forest,23766,SquareKilometer, +dcid:wikidataId/Q1185,2007,dcid:Area_LandCover_Forest,36190,SquareKilometer, +dcid:wikidataId/Q1185,2009,dcid:Area_LandCover_Forest,36190,SquareKilometer, +dcid:wikidataId/Q1185,2011,dcid:Area_LandCover_Forest,36194,SquareKilometer, +dcid:wikidataId/Q1185,2013,dcid:Area_LandCover_Forest,36171,SquareKilometer, +dcid:wikidataId/Q1185,2015,dcid:Area_LandCover_Forest,36449,SquareKilometer, +dcid:wikidataId/Q1185,2017,dcid:Area_LandCover_Forest,37550,SquareKilometer, +dcid:wikidataId/Q1185,2019,dcid:Area_LandCover_Forest,38575,SquareKilometer, +dcid:wikidataId/Q1185,2021,dcid:Area_LandCover_Forest,39107,SquareKilometer, +dcid:wikidataId/Q1185,2023,dcid:Area_LandCover_Forest,39254,SquareKilometer, +dcid:wikidataId/Q1186,2007,dcid:Area_LandCover_Forest,17324,SquareKilometer, +dcid:wikidataId/Q1186,2009,dcid:Area_LandCover_Forest,17324,SquareKilometer, +dcid:wikidataId/Q1186,2011,dcid:Area_LandCover_Forest,17300,SquareKilometer, +dcid:wikidataId/Q1186,2013,dcid:Area_LandCover_Forest,18384,SquareKilometer, +dcid:wikidataId/Q1186,2015,dcid:Area_LandCover_Forest,19278,SquareKilometer, +dcid:wikidataId/Q1186,2017,dcid:Area_LandCover_Forest,20321,SquareKilometer, +dcid:wikidataId/Q1186,2019,dcid:Area_LandCover_Forest,21144,SquareKilometer, +dcid:wikidataId/Q1186,2021,dcid:Area_LandCover_Forest,21926,SquareKilometer, +dcid:wikidataId/Q1186,2023,dcid:Area_LandCover_Forest,22059,SquareKilometer, +dcid:wikidataId/Q26927,2007,dcid:Area_LandCover_Forest,26,SquareKilometer, +dcid:wikidataId/Q26927,2009,dcid:Area_LandCover_Forest,26,SquareKilometer, +dcid:wikidataId/Q26927,2011,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q26927,2013,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q26927,2015,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q26927,2017,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q26927,2019,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q26927,2021,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q26927,2023,dcid:Area_LandCover_Forest,27,SquareKilometer, +dcid:wikidataId/Q1188,2007,dcid:Area_LandCover_Forest,77700,SquareKilometer, +dcid:wikidataId/Q1188,2009,dcid:Area_LandCover_Forest,77700,SquareKilometer, +dcid:wikidataId/Q1188,2011,dcid:Area_LandCover_Forest,77700,SquareKilometer, +dcid:wikidataId/Q1188,2013,dcid:Area_LandCover_Forest,77522,SquareKilometer, +dcid:wikidataId/Q1188,2015,dcid:Area_LandCover_Forest,77426,SquareKilometer, +dcid:wikidataId/Q1188,2017,dcid:Area_LandCover_Forest,77414,SquareKilometer, +dcid:wikidataId/Q1188,2019,dcid:Area_LandCover_Forest,77482,SquareKilometer, +dcid:wikidataId/Q1188,2021,dcid:Area_LandCover_Forest,77445,SquareKilometer, +dcid:wikidataId/Q1188,2023,dcid:Area_LandCover_Forest,77073,SquareKilometer, +dcid:wikidataId/Q1191,2007,dcid:Area_LandCover_Forest,50650,SquareKilometer, +dcid:wikidataId/Q1191,2009,dcid:Area_LandCover_Forest,50650,SquareKilometer, +dcid:wikidataId/Q1191,2011,dcid:Area_LandCover_Forest,50646,SquareKilometer, +dcid:wikidataId/Q1191,2013,dcid:Area_LandCover_Forest,50632,SquareKilometer, +dcid:wikidataId/Q1191,2015,dcid:Area_LandCover_Forest,50699,SquareKilometer, +dcid:wikidataId/Q1191,2017,dcid:Area_LandCover_Forest,50682,SquareKilometer, +dcid:wikidataId/Q1191,2019,dcid:Area_LandCover_Forest,50778,SquareKilometer, +dcid:wikidataId/Q1191,2021,dcid:Area_LandCover_Forest,50913,SquareKilometer, +dcid:wikidataId/Q1191,2023,dcid:Area_LandCover_Forest,50859,SquareKilometer, +dcid:wikidataId/Q1193,2007,dcid:Area_LandCover_Forest,17280,SquareKilometer, +dcid:wikidataId/Q1193,2009,dcid:Area_LandCover_Forest,17280,SquareKilometer, +dcid:wikidataId/Q1193,2011,dcid:Area_LandCover_Forest,17090,SquareKilometer, +dcid:wikidataId/Q1193,2013,dcid:Area_LandCover_Forest,16961,SquareKilometer, +dcid:wikidataId/Q1193,2015,dcid:Area_LandCover_Forest,17083,SquareKilometer, +dcid:wikidataId/Q1193,2017,dcid:Area_LandCover_Forest,17346,SquareKilometer, +dcid:wikidataId/Q1193,2019,dcid:Area_LandCover_Forest,16847,SquareKilometer, +dcid:wikidataId/Q1193,2021,dcid:Area_LandCover_Forest,16640,SquareKilometer, +dcid:wikidataId/Q1193,2023,dcid:Area_LandCover_Forest,16585,SquareKilometer, +dcid:wikidataId/Q1195,2007,dcid:Area_LandCover_Forest,17321,SquareKilometer, +dcid:wikidataId/Q1195,2009,dcid:Area_LandCover_Forest,17321,SquareKilometer, +dcid:wikidataId/Q1195,2011,dcid:Area_LandCover_Forest,17275,SquareKilometer, +dcid:wikidataId/Q1195,2013,dcid:Area_LandCover_Forest,17241,SquareKilometer, +dcid:wikidataId/Q1195,2015,dcid:Area_LandCover_Forest,17262,SquareKilometer, +dcid:wikidataId/Q1195,2017,dcid:Area_LandCover_Forest,17146,SquareKilometer, +dcid:wikidataId/Q1195,2019,dcid:Area_LandCover_Forest,17119,SquareKilometer, +dcid:wikidataId/Q1195,2021,dcid:Area_LandCover_Forest,16997,SquareKilometer, +dcid:wikidataId/Q1195,2023,dcid:Area_LandCover_Forest,16967,SquareKilometer, +dcid:wikidataId/Q1502,2007,dcid:Area_LandCover_Forest,19240,SquareKilometer, +dcid:wikidataId/Q1502,2009,dcid:Area_LandCover_Forest,19183,SquareKilometer, +dcid:wikidataId/Q1502,2011,dcid:Area_LandCover_Forest,19117,SquareKilometer, +dcid:wikidataId/Q1502,2013,dcid:Area_LandCover_Forest,18978,SquareKilometer, +dcid:wikidataId/Q1502,2015,dcid:Area_LandCover_Forest,18717,SquareKilometer, +dcid:wikidataId/Q1502,2017,dcid:Area_LandCover_Forest,18186,SquareKilometer, +dcid:wikidataId/Q1502,2019,dcid:Area_LandCover_Forest,18006,SquareKilometer, +dcid:wikidataId/Q1502,2021,dcid:Area_LandCover_Forest,17749,SquareKilometer, +dcid:wikidataId/Q1502,2023,dcid:Area_LandCover_Forest,17990,SquareKilometer, +dcid:wikidataId/Q1599,2007,dcid:Area_LandCover_Forest,13464,SquareKilometer, +dcid:wikidataId/Q1599,2009,dcid:Area_LandCover_Forest,13464,SquareKilometer, +dcid:wikidataId/Q1599,2011,dcid:Area_LandCover_Forest,13318,SquareKilometer, +dcid:wikidataId/Q1599,2013,dcid:Area_LandCover_Forest,13017,SquareKilometer, +dcid:wikidataId/Q1599,2015,dcid:Area_LandCover_Forest,12939,SquareKilometer, +dcid:wikidataId/Q1599,2017,dcid:Area_LandCover_Forest,12489,SquareKilometer, +dcid:wikidataId/Q1599,2019,dcid:Area_LandCover_Forest,12486,SquareKilometer, +dcid:wikidataId/Q1599,2021,dcid:Area_LandCover_Forest,12274,SquareKilometer, +dcid:wikidataId/Q1599,2023,dcid:Area_LandCover_Forest,12222,SquareKilometer, +dcid:wikidataId/Q22048,2007,dcid:Area_LandCover_Forest,48855,SquareKilometer, +dcid:wikidataId/Q22048,2009,dcid:Area_LandCover_Forest,48855,SquareKilometer, +dcid:wikidataId/Q22048,2011,dcid:Area_LandCover_Forest,48903,SquareKilometer, +dcid:wikidataId/Q22048,2013,dcid:Area_LandCover_Forest,50545,SquareKilometer, +dcid:wikidataId/Q22048,2015,dcid:Area_LandCover_Forest,50460,SquareKilometer, +dcid:wikidataId/Q22048,2017,dcid:Area_LandCover_Forest,51345,SquareKilometer, +dcid:wikidataId/Q22048,2019,dcid:Area_LandCover_Forest,51619,SquareKilometer, +dcid:wikidataId/Q22048,2021,dcid:Area_LandCover_Forest,52282,SquareKilometer, +dcid:wikidataId/Q22048,2023,dcid:Area_LandCover_Forest,52434,SquareKilometer, +dcid:wikidataId/Q66743,2007,dcid:Area_LandCover_Forest,44,SquareKilometer, +dcid:wikidataId/Q66743,2009,dcid:Area_LandCover_Forest,50,SquareKilometer, +dcid:wikidataId/Q66743,2011,dcid:Area_LandCover_Forest,50,SquareKilometer, +dcid:wikidataId/Q66743,2013,dcid:Area_LandCover_Forest,51,SquareKilometer, +dcid:wikidataId/Q66743,2015,dcid:Area_LandCover_Forest,57,SquareKilometer, +dcid:wikidataId/Q66743,2017,dcid:Area_LandCover_Forest,54,SquareKilometer, +dcid:wikidataId/Q66743,2019,dcid:Area_LandCover_Forest,52,SquareKilometer, +dcid:wikidataId/Q66743,2021,dcid:Area_LandCover_Forest,44,SquareKilometer, +dcid:wikidataId/Q66743,2023,dcid:Area_LandCover_Forest,44,SquareKilometer, +dcid:wikidataId/Q22424,2007,dcid:Area_LandCover_Forest,1664,SquareKilometer, +dcid:wikidataId/Q22424,2009,dcid:Area_LandCover_Forest,1664,SquareKilometer, +dcid:wikidataId/Q22424,2011,dcid:Area_LandCover_Forest,1764,SquareKilometer, +dcid:wikidataId/Q22424,2013,dcid:Area_LandCover_Forest,1791,SquareKilometer, +dcid:wikidataId/Q22424,2015,dcid:Area_LandCover_Forest,1771,SquareKilometer, +dcid:wikidataId/Q22424,2017,dcid:Area_LandCover_Forest,1837,SquareKilometer, +dcid:wikidataId/Q22424,2019,dcid:Area_LandCover_Forest,1849,SquareKilometer, +dcid:wikidataId/Q22424,2021,dcid:Area_LandCover_Forest,1847,SquareKilometer, +dcid:wikidataId/Q22424,2023,dcid:Area_LandCover_Forest,1846,SquareKilometer, +dcid:wikidataId/Q1437,2007,dcid:Area_LandCover_Forest,16036,SquareKilometer, +dcid:wikidataId/Q1437,2009,dcid:Area_LandCover_Forest,16036,SquareKilometer, +dcid:wikidataId/Q1437,2011,dcid:Area_LandCover_Forest,16087,SquareKilometer, +dcid:wikidataId/Q1437,2013,dcid:Area_LandCover_Forest,16086,SquareKilometer, +dcid:wikidataId/Q1437,2015,dcid:Area_LandCover_Forest,16106,SquareKilometer, +dcid:wikidataId/Q1437,2017,dcid:Area_LandCover_Forest,16572,SquareKilometer, +dcid:wikidataId/Q1437,2019,dcid:Area_LandCover_Forest,16630,SquareKilometer, +dcid:wikidataId/Q1437,2021,dcid:Area_LandCover_Forest,16632,SquareKilometer, +dcid:wikidataId/Q1437,2023,dcid:Area_LandCover_Forest,16548,SquareKilometer, +dcid:wikidataId/Q1505,2007,dcid:Area_LandCover_Forest,3357,SquareKilometer, +dcid:wikidataId/Q1505,2009,dcid:Area_LandCover_Forest,3359,SquareKilometer, +dcid:wikidataId/Q1505,2011,dcid:Area_LandCover_Forest,3359,SquareKilometer, +dcid:wikidataId/Q1505,2013,dcid:Area_LandCover_Forest,3358,SquareKilometer, +dcid:wikidataId/Q1505,2015,dcid:Area_LandCover_Forest,3353,SquareKilometer, +dcid:wikidataId/Q1505,2017,dcid:Area_LandCover_Forest,3344,SquareKilometer, +dcid:wikidataId/Q1505,2019,dcid:Area_LandCover_Forest,3342,SquareKilometer, +dcid:wikidataId/Q1505,2021,dcid:Area_LandCover_Forest,3353,SquareKilometer, +dcid:wikidataId/Q1505,2023,dcid:Area_LandCover_Forest,3358,SquareKilometer, +dcid:wikidataId/Q1445,2007,dcid:Area_LandCover_Forest,23338,SquareKilometer, +dcid:wikidataId/Q1445,2009,dcid:Area_LandCover_Forest,23551,SquareKilometer, +dcid:wikidataId/Q1445,2011,dcid:Area_LandCover_Forest,23625,SquareKilometer, +dcid:wikidataId/Q1445,2013,dcid:Area_LandCover_Forest,24245,SquareKilometer, +dcid:wikidataId/Q1445,2015,dcid:Area_LandCover_Forest,26208,SquareKilometer, +dcid:wikidataId/Q1445,2017,dcid:Area_LandCover_Forest,26281,SquareKilometer, +dcid:wikidataId/Q1445,2019,dcid:Area_LandCover_Forest,26364,SquareKilometer, +dcid:wikidataId/Q1445,2021,dcid:Area_LandCover_Forest,26511,SquareKilometer, +dcid:wikidataId/Q1445,2023,dcid:Area_LandCover_Forest,26450,SquareKilometer, +dcid:wikidataId/Q677037,2013,dcid:Area_LandCover_Forest,19957,SquareKilometer, +dcid:wikidataId/Q677037,2015,dcid:Area_LandCover_Forest,19854,SquareKilometer, +dcid:wikidataId/Q677037,2017,dcid:Area_LandCover_Forest,20419,SquareKilometer, +dcid:wikidataId/Q677037,2019,dcid:Area_LandCover_Forest,20582,SquareKilometer, +dcid:wikidataId/Q677037,2021,dcid:Area_LandCover_Forest,21279,SquareKilometer, +dcid:wikidataId/Q677037,2023,dcid:Area_LandCover_Forest,21179,SquareKilometer, +dcid:wikidataId/Q1363,2007,dcid:Area_LandCover_Forest,8073,SquareKilometer, +dcid:wikidataId/Q1363,2009,dcid:Area_LandCover_Forest,7985,SquareKilometer, +dcid:wikidataId/Q1363,2011,dcid:Area_LandCover_Forest,7977,SquareKilometer, +dcid:wikidataId/Q1363,2013,dcid:Area_LandCover_Forest,7949,SquareKilometer, +dcid:wikidataId/Q1363,2015,dcid:Area_LandCover_Forest,7890,SquareKilometer, +dcid:wikidataId/Q1363,2017,dcid:Area_LandCover_Forest,7726,SquareKilometer, +dcid:wikidataId/Q1363,2019,dcid:Area_LandCover_Forest,7726,SquareKilometer, +dcid:wikidataId/Q1363,2021,dcid:Area_LandCover_Forest,7680,SquareKilometer, +dcid:wikidataId/Q1363,2023,dcid:Area_LandCover_Forest,7585,SquareKilometer, +dcid:wikidataId/Q1498,2007,dcid:Area_LandCover_Forest,14341,SquareKilometer, +dcid:wikidataId/Q1498,2009,dcid:Area_LandCover_Forest,14341,SquareKilometer, +dcid:wikidataId/Q1498,2011,dcid:Area_LandCover_Forest,14338,SquareKilometer, +dcid:wikidataId/Q1498,2013,dcid:Area_LandCover_Forest,14360,SquareKilometer, +dcid:wikidataId/Q1498,2015,dcid:Area_LandCover_Forest,14401,SquareKilometer, +dcid:wikidataId/Q1498,2017,dcid:Area_LandCover_Forest,14679,SquareKilometer, +dcid:wikidataId/Q1498,2019,dcid:Area_LandCover_Forest,14806,SquareKilometer, +dcid:wikidataId/Q1498,2021,dcid:Area_LandCover_Forest,14927,SquareKilometer, +dcid:wikidataId/Q1498,2023,dcid:Area_LandCover_Forest,15046,SquareKilometer, +dcid:wikidataId/Q1499,2007,dcid:Area_LandCover_Forest,24495,SquareKilometer, +dcid:wikidataId/Q1499,2009,dcid:Area_LandCover_Forest,24495,SquareKilometer, +dcid:wikidataId/Q1499,2011,dcid:Area_LandCover_Forest,24496,SquareKilometer, +dcid:wikidataId/Q1499,2013,dcid:Area_LandCover_Forest,24508,SquareKilometer, +dcid:wikidataId/Q1499,2015,dcid:Area_LandCover_Forest,24272,SquareKilometer, +dcid:wikidataId/Q1499,2017,dcid:Area_LandCover_Forest,24295,SquareKilometer, +dcid:wikidataId/Q1499,2019,dcid:Area_LandCover_Forest,24303,SquareKilometer, +dcid:wikidataId/Q1499,2021,dcid:Area_LandCover_Forest,24327,SquareKilometer, +dcid:wikidataId/Q1499,2023,dcid:Area_LandCover_Forest,24304,SquareKilometer, +dcid:wikidataId/Q1356,2007,dcid:Area_LandCover_Forest,12994,SquareKilometer, +dcid:wikidataId/Q1356,2009,dcid:Area_LandCover_Forest,12994,SquareKilometer, +dcid:wikidataId/Q1356,2011,dcid:Area_LandCover_Forest,12995,SquareKilometer, +dcid:wikidataId/Q1356,2013,dcid:Area_LandCover_Forest,17102,SquareKilometer, +dcid:wikidataId/Q1356,2015,dcid:Area_LandCover_Forest,16826,SquareKilometer, +dcid:wikidataId/Q1356,2017,dcid:Area_LandCover_Forest,16847,SquareKilometer, +dcid:wikidataId/Q1356,2019,dcid:Area_LandCover_Forest,16902,SquareKilometer, +dcid:wikidataId/Q1356,2021,dcid:Area_LandCover_Forest,16782,SquareKilometer, +dcid:wikidataId/Q1356,2023,dcid:Area_LandCover_Forest,16832,SquareKilometer, +dcid:wikidataId/Q200667,2013,dcid:Area_LandCover_Forest,1478,SquareKilometer, +dcid:wikidataId/Q200667,2021,dcid:Area_LandCover_Forest,2279,SquareKilometer, +dcid:wikidataId/Q200667,2023,dcid:Area_LandCover_Forest,2286,SquareKilometer, +dcid:country/IND,2007,dcid:Area_LandCover_Forest,690899,SquareKilometer, +dcid:country/IND,2009,dcid:Area_LandCover_Forest,692394,SquareKilometer, +dcid:country/IND,2011,dcid:Area_LandCover_Forest,692027,SquareKilometer, +dcid:country/IND,2013,dcid:Area_LandCover_Forest,698712,SquareKilometer, +dcid:country/IND,2015,dcid:Area_LandCover_Forest,701495,SquareKilometer, +dcid:country/IND,2017,dcid:Area_LandCover_Forest,708273,SquareKilometer, +dcid:country/IND,2019,dcid:Area_LandCover_Forest,712249,SquareKilometer, +dcid:country/IND,2021,dcid:Area_LandCover_Forest,715186,SquareKilometer, +dcid:country/IND,2023,dcid:Area_LandCover_Forest,715343,SquareKilometer, From 31a1173bc2757f121cddbba3048b79a302d9ca7a Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 21 Sep 2026 13:47:46 +0000 Subject: [PATCH 11/20] Modified changes as per Internal CRA report --- .../rbi_download.py | 5 ++- .../india_rbi_state_statistics/run.sh | 38 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 206ace20c9..e2e5688812 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -315,8 +315,9 @@ def is_state_header(val): if pd.isna(val): return False return bool( - re.search(r'state\s*/\s*union\s*territory', str(val), - re.IGNORECASE)) + re.search( + r'state\s*/\s*(?:union(?:\s*territory)?|u\.?t\.?\b)', + str(val), re.IGNORECASE)) for sheet_name, df in all_sheets_data.items(): df = _apply_map(df, clean_cell) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index 5415a0d12b..3fe8bcc40e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -2,7 +2,7 @@ set -e SCRIPT_PATH=$(realpath "$(dirname "$0")") -COUNTERS_PATH=$SCRIPT_PATH/counters +COUNTERS_PATH="$SCRIPT_PATH/counters" mkdir -p "$COUNTERS_PATH" # Existing statvar MCF path with local fallback @@ -13,41 +13,41 @@ else EXISTING_MCF="${EXISTING_STATVAR_MCF:-gs://unresolved_mcf/scripts/statvar/stat_vars.mcf}" fi -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/agriculture/*.xlsx" --pv_map=$SCRIPT_PATH/agriculture_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/agriculture_counters.csv --output_path=$SCRIPT_PATH/output_files/agriculture/agriculture_output || { echo "Error: Processing Agriculture files failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/agriculture/*.xlsx" --pv_map="$SCRIPT_PATH/agriculture_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/agriculture_counters.csv" --output_path="$SCRIPT_PATH/output_files/agriculture/agriculture_output" || { echo "Error: Processing Agriculture files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_forest_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_forest_cover_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output" || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_tree_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_tree_cover_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output" || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/sub_division_wise_annual_rainfall_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/sub_division_wise_annual_rainfall_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output" || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output" || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/environment_sdg_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_sdg_score_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/environment_sdg_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_sdg_score_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output" || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/price_and_wages/*.xlsx" --pv_map=$SCRIPT_PATH/price_wages_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/price_and_wages_counters.csv --output_path=$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output || { echo "Error: Processing price_and_wages files failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/price_and_wages/*.xlsx" --pv_map="$SCRIPT_PATH/price_wages_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/price_and_wages_counters.csv" --output_path="$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output" || { echo "Error: Processing price_and_wages files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_per_capita_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_per_capita_availability_of_power_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output" || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_availability_of_power_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output" || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_installed_capacity_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_installed_capacity_of_power_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output" || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_power_requirement_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_power_requirement_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output" || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_national_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_length_of_national_highways_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output" || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_railway_route_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_railway_route_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output" || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_roads_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output || { echo "Error: Processing State-wise Length of Roads failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_length_of_roads_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output" || { echo "Error: Processing State-wise Length of Roads failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_state_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_length_of_state_highways_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output" || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_electricity_transmission_distribution_losses_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_electricity_transmission_distribution_losses_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output" || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_telephones_per_100_population_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_telephones_per_100_population_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output" || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } +python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output" || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } EXIT_STATUS=$? echo "All processing steps completed successfully." From 1722471d35c7429a31ec54b04635a092d3731767 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Tue, 22 Sep 2026 14:39:52 +0000 Subject: [PATCH 12/20] Modifications as per CRA internal review --- .../india_rbi_state_statistics/README.md | 7 +- .../agriculture_pvmap.csv | 2 +- .../rbi_download.py | 6 +- .../rbi_download_test.py | 15 +- .../india_rbi_state_statistics/run.sh | 248 ++++++++++++++++-- .../testdata/environment_input.xlsx | Bin 11532 -> 12331 bytes .../testdata/environment_output.csv | 16 ++ .../validation_config.json | 2 +- 8 files changed, 268 insertions(+), 28 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 9c74cd9d80..568739da96 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -345,7 +345,12 @@ This import uses `validation_config.json` with: 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q9357528` (439 records in agriculture, 13 records in relief expenditure). 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, where Import Differ classifies old unit nodes as deleted and new unit nodes as added). 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations (`Dadra & Nagar Haveli and Daman & Diu`). - - **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). + - **Technical Breakdown of Deletions**: + | Category | Output File | Deletion % / Records | Root Cause | + |---|---|---|---| + | Environment | `sub_division_wise_annual_rainfall_output.csv` | 6.94% (24 records) | Remapped `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana state) to dedicated IMD sub-division `dcid:HaryanaDelhiChandigarh`. | + | Agriculture | `agriculture_output.csv` | 5.64% (1,282 records) | Corrected `NCT of Delhi` to `wikidataId/Q9357528` (439 records); updated `Annual_Amount_FarmInventory_Fruits` unit to `MetricTon` (309 records); retrospective RBI revisions for provisional years `2022-03`–`2024-03` (534 records). | + | Environment | `state_wise_expenditure_on_relief_on_natural_calamities_output.csv` | 3.15% (13 records) | Corrected `NCT of Delhi` place resolution from `wikidataId/Q1352` (Chennai) to `wikidataId/Q9357528`. | 2. **`SQL_VALIDATOR` non-empty check (`check_expected_statvar_count`)**: - Asserts `statvar_cnt >= 1` so an empty `summary_report.csv` cannot vacuously pass grouped SQL queries. 3. **`SQL_VALIDATOR` cadence freshness check (`check_max_date_freshness`)**: diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/agriculture_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/agriculture_pvmap.csv index 1110d3b0e4..fff34a4a85 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/agriculture_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/agriculture_pvmap.csv @@ -81,4 +81,4 @@ TABLE 94: STATE-WISE WOOL PRODUCTION,populationType,FarmInventory,measuredProper 2029-30,value,{Number},observationDate,2030-03,observationPeriod,P1Y,,,,,, 2030-31,value,{Number},observationDate,2031-03,observationPeriod,P1Y,,,,,, 2031-32,value,{Number},observationDate,2032-03,observationPeriod,P1Y,,,,,, -2032-33,value,{Number},observationDate,2033-03,observationPeriod,P1Y,,,,,, \ No newline at end of file +2032-33,value,{Number},observationDate,2033-03,observationPeriod,P1Y,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index e2e5688812..6a79eb1d9c 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -359,7 +359,9 @@ def is_state_header(val): return [] -def main(_): +def main(argv): + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') configs = reads_config_file() RBI_URL = configs['URLS_CONFIG'] failed_downloads = download_files(RBI_URL) @@ -367,7 +369,6 @@ def main(_): logging.fatal( f"Download failed for {len(failed_downloads)} table(s): {failed_downloads}" ) - return logging.info("Download process Completed successfully") directories = [ 'agriculture', 'environment', 'infrastructure', 'price_and_wages' @@ -381,7 +382,6 @@ def main(_): logging.fatal( f"Preprocessing failed for {len(failed_preprocesses)} file(s): {failed_preprocesses}" ) - return logging.info("Pre-process Completed successfully") diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index ce1be2c66c..161ad31b36 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -19,6 +19,7 @@ import tempfile from unittest import mock +from absl import app from absl.testing import absltest, flagsaver import pandas as pd import requests @@ -365,8 +366,9 @@ def test_main_fatal_when_download_fails(self): rbi_download, 'reads_config_file', return_value={'URLS_CONFIG': []}), \ mock.patch.object( rbi_download, 'download_files', return_value=['failed_table.xlsx']), \ - mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: - rbi_download.main([]) + mock.patch.object(rbi_download.logging, 'fatal', side_effect=SystemExit) as mock_fatal: + with self.assertRaises(SystemExit): + rbi_download.main(['rbi_download']) mock_fatal.assert_called_once() self.assertIn('failed_table.xlsx', mock_fatal.call_args[0][0]) @@ -377,11 +379,16 @@ def test_main_fatal_when_preprocessing_fails(self): rbi_download, 'download_files', return_value=[]), \ mock.patch.object( rbi_download, 'preprocess_files', side_effect=[['broken.xlsx'], [], [], []]), \ - mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: - rbi_download.main([]) + mock.patch.object(rbi_download.logging, 'fatal', side_effect=SystemExit) as mock_fatal: + with self.assertRaises(SystemExit): + rbi_download.main(['rbi_download']) mock_fatal.assert_called_once() self.assertIn('broken.xlsx', mock_fatal.call_args[0][0]) + def test_main_usage_error_too_many_args(self): + with self.assertRaises(app.UsageError): + rbi_download.main(['rbi_download', 'unexpected_argument']) + if __name__ == '__main__': absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index 3fe8bcc40e..7a41a7da12 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -13,41 +13,253 @@ else EXISTING_MCF="${EXISTING_STATVAR_MCF:-gs://unresolved_mcf/scripts/statvar/stat_vars.mcf}" fi -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/agriculture/*.xlsx" --pv_map="$SCRIPT_PATH/agriculture_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/agriculture_counters.csv" --output_path="$SCRIPT_PATH/output_files/agriculture/agriculture_output" || { echo "Error: Processing Agriculture files failed!"; exit 1; } +PROCESSOR="$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" +IN_DIR="$SCRIPT_PATH/input_files" +OUT_DIR="$SCRIPT_PATH/output_files" +IN_ENV="$IN_DIR/environment" +IN_INFRA="$IN_DIR/infrastructure" +OUT_ENV="$OUT_DIR/environment" +OUT_INFRA="$OUT_DIR/infrastructure" +PLACES_CSV="$SCRIPT_PATH/rbi_places_resolver.csv" +META_RBI="$SCRIPT_PATH/rbi_metadata.csv" +META_INFRA="$SCRIPT_PATH/infrastructure_metadata.csv" +META_SDG="$SCRIPT_PATH/environment_sdg_metadata.csv" -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_forest_cover_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output" || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_DIR/agriculture/*.xlsx" \ + --pv_map="$SCRIPT_PATH/agriculture_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/agriculture_counters.csv" \ + --output_path="$OUT_DIR/agriculture/agriculture_output" \ + || { + echo "Error: Processing Agriculture files failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_tree_cover_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output" || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_ENV/state_wise_forest_cover.xlsx" \ + --pv_map="$SCRIPT_PATH/environment_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_forest_cover_counters.csv" \ + --output_path="$OUT_ENV/state_wise_forest_cover_output" \ + || { + echo "Error: Processing State-wise Forest Cover failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/sub_division_wise_annual_rainfall_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output" || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_ENV/state_wise_tree_cover.xlsx" \ + --pv_map="$SCRIPT_PATH/environment_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_tree_cover_counters.csv" \ + --output_path="$OUT_ENV/state_wise_tree_cover_output" \ + || { + echo "Error: Processing State-wise Tree Cover failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output" || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_ENV/sub_division_wise_annual_rainfall.xlsx" \ + --pv_map="$SCRIPT_PATH/environment_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/sub_division_wise_annual_rainfall_counters.csv" \ + --output_path="$OUT_ENV/sub_division_wise_annual_rainfall_output" \ + || { + echo "Error: Processing Sub-Division-wise Annual Rainfall failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" --pv_map="$SCRIPT_PATH/environment_pvmap.csv" --config_file="$SCRIPT_PATH/environment_sdg_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_sdg_score_counters.csv" --output_path="$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output" || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_ENV/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" \ + --pv_map="$SCRIPT_PATH/environment_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters=\ + "$COUNTERS_PATH/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv" \ + --output_path="$OUT_ENV/state_wise_expenditure_on_relief_on_natural_calamities_output" \ + || { + echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/price_and_wages/*.xlsx" --pv_map="$SCRIPT_PATH/price_wages_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/price_and_wages_counters.csv" --output_path="$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output" || { echo "Error: Processing price_and_wages files failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_ENV/state_wise_sustainable_development_goals_score_SDGs.xlsx" \ + --pv_map="$SCRIPT_PATH/environment_pvmap.csv" \ + --config_file="$META_SDG" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_sdg_score_counters.csv" \ + --output_path="$OUT_ENV/state_wise_sdg_score_output" \ + || { + echo "Error: Processing State-wise Sustainable Development Goals Score failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_per_capita_availability_of_power_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output" || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_DIR/price_and_wages/*.xlsx" \ + --pv_map="$SCRIPT_PATH/price_wages_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/price_and_wages_counters.csv" \ + --output_path="$OUT_DIR/price_and_wages/price_and_wages_output" \ + || { + echo "Error: Processing price_and_wages files failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_availability_of_power_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output" || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_per_capita_availability_of_power.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_per_capita_availability_of_power_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_per_capita_availability_of_power_output" \ + || { + echo "Error: Processing State-wise Per Capita Availability of Power failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_installed_capacity_of_power_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output" || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_availability_of_power.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_availability_of_power_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_availability_of_power_output" \ + || { + echo "Error: Processing State-wise Availability of Power files failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_power_requirement_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output" || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_installed_capacity_of_power.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_installed_capacity_of_power_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_installed_capacity_of_power_output" \ + || { + echo "Error: Processing State-wise Installed Capacity of Power failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_length_of_national_highways_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output" || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_power_requirement.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_power_requirement_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_power_requirement_output" \ + || { + echo "Error: Processing State-wise Power Requirement failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_railway_route_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output" || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_length_of_national_highways.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_INFRA" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_length_of_national_highways_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_length_of_national_highways_output" \ + || { + echo "Error: Processing State-wise Length of National Highways failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_length_of_roads_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output" || { echo "Error: Processing State-wise Length of Roads failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_railway_route.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_INFRA" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_railway_route_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_railway_route_output" \ + || { + echo "Error: Processing State-wise Railway Route failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/infrastructure_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_length_of_state_highways_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output" || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_length_of_roads.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_INFRA" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_length_of_roads_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_length_of_roads_output" \ + || { + echo "Error: Processing State-wise Length of Roads failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_electricity_transmission_distribution_losses_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output" || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_length_of_state_highways.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_INFRA" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_length_of_state_highways_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_length_of_state_highways_output" \ + || { + echo "Error: Processing State-wise Length of State Highways failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_telephones_per_100_population_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output" || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_electricity_transmission_distribution_losses.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters=\ + "$COUNTERS_PATH/state_wise_electricity_transmission_distribution_losses_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_electricity_transmission_distribution_losses_output" \ + || { + echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!" + exit 1 + } -python3 "$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" --config_file="$SCRIPT_PATH/rbi_metadata.csv" --places_resolved_csv="$SCRIPT_PATH/rbi_places_resolver.csv" --existing_statvar_mcf="$EXISTING_MCF" --output_counters="$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv" --output_path="$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output" || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_telephones_per_100_population.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_telephones_per_100_population_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_telephones_per_100_population_output" \ + || { + echo "Error: Processing State-wise Telephones per 100 Population failed!" + exit 1 + } + +python3 "$PROCESSOR" \ + --input_data="$IN_INFRA/state_wise_road_constructed_under_PMGSY.xlsx" \ + --pv_map="$SCRIPT_PATH/infrastructure_pvmap.csv" \ + --config_file="$META_RBI" \ + --places_resolved_csv="$PLACES_CSV" \ + --existing_statvar_mcf="$EXISTING_MCF" \ + --output_counters="$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv" \ + --output_path="$OUT_INFRA/state_wise_road_constructed_under_pmgsy_output" \ + || { + echo "Error: Processing State-wise Road Constructed under PMGSY failed!" + exit 1 + } EXIT_STATUS=$? echo "All processing steps completed successfully." diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/testdata/environment_input.xlsx b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/testdata/environment_input.xlsx index 23cb3f9b2efefaeebf7946a8b64fea4b46cd3d62..6acf6eed9c252828ac509346401d1bba31637017 100644 GIT binary patch delta 9965 zcmZu%1ymeMvtBGfAd3d~5G1&}1ozJ7%Z}%C9BC{_Rg{(-`4+a&nr8PEojy zQ8sf7=(ZQ<-^_N!evy4X7KeL%7K*km>ODT z@I^-jM(nPI9m!XguZ$Q?nEcab)w>8HPr!eycG-hsD~AaJxtYV7@QGn>&`<#bmw6e2 z;D@J*SO!g4daHi@KS~mn7goD18T(IkgQdHh8K~Y?Tuieu{0sznWM2x;;$L@x-daEk zti_FeXFamPy9GOh4~q`&VLH~kt!)oix3`cchuY1L{;~PPsNF|7IP&$+=q}IGKmt)~MXe(3`+FB2=>A8y{4%b~i#)jNu!} zD#j0$x4wrX)e#xicmfQ9cpO(lle>a?-A5x)yXmFr8--$#yKfWjuSFYL^_s8FE;dBI zrj_r(s?eVUSOfF~F4-)baOR$khvxg;4%ea2wtP4C;~qn8@rAB2EN;hA^l#jbmFP#@ zjwR^X+>RCKD*#>5qoj)=jS7)YlZyh!VPKxzae_~Iy5_F+Ceru#Tad+1EX_fUjMu78@Kape9s_2q_3fS8@L6mS zt82dK^HYk(iH4=~pEa!8My^#D2XFk#-3zeHrzKK_rct+j?Mko^b?R4wtaDKWb2+X| zzX5L61P6lEwqUyrW*P4)%FlJ)@&Tg}KYdbd0yIIsr|+!DD(&AcXCShQ<^>{Vt#|jC zlA$eN3q&Jj#po9?t_YR0cryE*>lU35!A#t+gj||ls^*@t1LTMqY>4% zb3KHAiDQ3K;lTr;A~b;7T^RVWy|&2|z<@3VT$c%}P zZYAp0uiR87{Z<;b@tBcm!hfy+1xa&>M>yL>BWu$kO4dRy@&I zZv=vgF>CvVwyee_(yKBtrBS&_Cc9pHeV!;(fYNQNy@oNJg?y3~@$NsX#ILAd>L4dt z2B>D=M}M`qOi?}?Aat7k3iaPNXWsAThD+tEXu+dP)KFoi6CrRpg%TM+M^bvSe*C%v z`X+NDM#UVR*9CW+Eq#dOLM7VYiQuF9sH_U}MOw2F(68tm<$k5V%)@>nU0 z6|aXdXJZ6YeqJiV4(AsyXB1eSAMJlm-#pVLvo6KX1};<~UH35f(53-(Xsoab^Wp5d zgizQQPtYqsOD3!}DRt|c*SrnCa*1S3GZim^p+u3+?b*F(^nkV6ieog-ll@vwr^4&z z#rwT(a7fsVQm|u+K8+e)vG^r~JtYEpQ2r(2c|QuhA;xC^OL}2K6$ee+)m*y*f^aJA zgO3*2j!ZDm61E}t8wd%5_GaZ$Eqhwwt*U}<{nA!2VeYE?XFBzWL&yUiHtYUs9`DLR zjogYv3~pBpIWTOjDCK`6puqo{Hs5qOZCvO_MizXaW=E8~+IxdXp~F-nUqutHnLn%v z&nO`m3OBoB!>Kf87Fdnf!a>9tmMo(;c|^^j$k`;(g3ynZEI8Bno|2SK z(pvDd-E=7$2<4w|chMp)O*#i0ni}cIA^$c}P(fDkjMhRA-s-5qL0r{M*{I>@_xQ#% z_Mz2Qu)0IWBHUi!aLl+QLP|LgakY`XO$V<*t{ovjiXZYL9IrJ_kvk$y)|5V1jmDMc zohD2dOB`6!j07XmVJx+!Ag#y5N5|SKi1;Wcuqq?! zJ;D{??`Ak0(H<&AjosnvwB2u?gmx6!s9^IV@E#~*Ts2r``JLHlpMe5Nss?F3O#Pm_ zK}|DL?V(_GOfy51!tU^*<4I6JR;u9)XlFWg-Wjy#$DqRAZw3hpJPrwjKqVZ*eg%cN zW}?TpzzqJ)iJ)C=7*X|Ze$zn&6V$q998>sXNqAkzD+YBM!B9U7O$zxg*<`j0`;Ce`}Ne93rL|sphtKdWw1tszoHyh0K`-?SrIwQ8y8LZ zl0jg#@p6_X0F8N}7`?l>K3&vaOa-z2hdij{!EKUa>Tq8kXW7jZsLl%}6olYs%OPj{ zR6ywrrBtQEfhfxjM^J^nSeLNr>hyS{B8BrCg((1H#-o8D#cu-P;hE!ST6tv+pcLIN zcEMNO7hNyqp^+W8O8-T#VLq!SJh8+~&QMJtL*+|@Vaeqp$B{>JgxF-8&&=IBQE@B| z+EApcmy`4LV0mOvZMi;BB&j8{VCYS^B`+cST7n6hm4Sx24Ra;V0G;qKOP#YpHzeR${A0H zxD>S^&=JudQl0sn-uL+M%V;Q1-NS_4x{`ls2KuM#-ShE60x--3mqeu~L_(03KNcT! zX=?Z#fnFSHGKAnk(WJD;BXeKQd0co+T5Zo4Wit3w)KiM%=B9$5-zysTb#!6xq|__2 zWRzZ{4^r(%;;c9jRb4q`Ls%~^4WJjN%xJh^HxZ$bSu*iKA@ea7D;SkbBr6y_6nVpQ zI0ihD^+YF^U3=wU{HAIQ6s1!`M)AFzaQ%#;j_x0jO3;S_63teFPz4*A$p_@n2b&S1 z&|oaRwgb_sydTp3olQq7e}m0Ejs274H+n z(F!h;Dp_tV#X{uT9kbViNhQQ|G7cdjIvBx1ju!Abj_)4HjG~ib*F&Svr;@rnq+zNo$4KAa!Ua7=e{;Co0 zK6$rVJk-;-q#<*z3DnHqYW~({VeV&m?x~IZBH*R$0z;?b6$-8UstwC?9CO~Cb_rM3 z27gQsJ))MUa@#A8o2zniB|Qq_G=#&_lVlMrJ`__T4g0s>9SO*Qc&*hhJNt8sW>xP* z!9HK_$-zP6`#XH#EaTIgn_f!S%gTeByYr*AU8|z*#$3~xd#AR8wif>T?#>f7Wy5>Q ze=Y>RQ{g8Xw-IdarUdd`R(H22TVEKXf0}H3t3bf zRcKwDw(8f!9jFY#C@Fo54^r+9J9e&STN+EtOYV2?wp1Qmy=MkK8|w@eIFRO+IxJ@z zra3Collf<^jWl{M+@1m!^-1ZqccQoPY)hGgp|3-t7ao@(Ts9%M1*r+hW&W1*Q25%w-ZlXPR*OXt(XSbVly}64%6t zG!?id1~~kh-lt#L9&LJTP|WH{ae6Vs=sCS!!K^vGSYd>mURUN<-6H25BDI4eNw;;S z5B6F*#Fj0#(^|_~qqJXxr2k(5+F>kMvj^IJi4N55x`x=hf45{8fVb{tjk4W)p-Rd= zK+2n-55G@Ww@gldZbBLbvK)Aab~ZO`sznwvDD2$})2nlbWD8R$?L`G^%o}F~1#6CN zPCRX4Y;z(DG3CD<@-_&EB)(`_+{%7Ac(C&=+1iVNNBdT*519w~iA}@`k5rFlGEVg7 zE!B~^I*whh);;V3L-M;uc_TJ0LBnF!r@W0~?^^b9Nt1rg50b;lqu2YRCHrffQZhM3 zJ#5wNQ~DV3cj`AW^+S+_N3y9aM*Ycy`tm@NH2g#DI44A}X!Y^{QrgZ@la z)nBA~gB(UAw5Rc3dug_4jpN*HaMn)wEG;kp? z(<&E`hOm;FGU~tE(8q{rn!9WvgBTznDckq1z1H-m6JZ%pj4d#Ev%Q*iw0Jhw;*9}KnL66^m;vD79lavfcX8E z@gV6*R?@w5gC^9(Q7%mN)9#Z>xfr6(6E#xGYGg8Gz^NOvDTL0DkO!Aoc4aI@h>4${ zsorr~Myel&ZoWRcs)%t=S{|B5;4Sx!Qh~{y`h^D8WJ##ZXLs|C3OZc4MUH6B8NpXK zh25j}@SheZeWxfc_uIZdQg;h$iBD8Dh{6-Ew+oT!O6o>g@T1CEaEmPx1gR%0augYO zxV7kDz+-Ev1<#K2;l&~uKPfcv6=s9RWG8JS(y=4ip@!vsryRMxQ&eXN4iYvZ`vGzy zWUarVDUyGMsG77Iikeh~ifFJ0D@I2UtzFWkk=P76E1RV$ z!ZB)$9WQb}tJ|mSl2|p-Ry0k4;$m$&L-rtpxHFQp92n#a)<1jE%zP4m2HOJyMU4Z* zok72hUgH(~>L^R4ejirqR>rYc6a0>?m@TE`MwGU-Pk`1^4`ILdu|wF@WUeo{%~0tw zB+bzXd%h6Mw#!dtP=_(lu=GNc z(REHSv?q85m##bSK6k99l0xQ;d_SFGIMb`rBJK|adL@HeJc!mjclwruFcApb-jX3? z#&ZQ@-LwT#O8|J_Tq2s4n*t8b+Ta9^;)kM-*M1mb%d{5P1cO5~95`Ivm;@u}%8zkG z(QK25{j{HGEj3_&iSJYo1%e{oft?jp-uxCV)o9NyCZXzObhP|kFWjNnZk2vvE#53Y z&aP8VsT@mgYHRSH635;{;c}kt6O5>+K`z3|6xq#=g7?38ZuV5R{?K zDdo8!_y)=q+LOZ0O?TE+Qu;jkLTwaVX>GMGDV(^IcU9`S-3sa;upRarw#}8U` zuELk&2Z59MK~`LtAfMMGFt-qgqR1KZ1$YH=YYh7!L=8$IS9?CkqD+$AI8K%TQ50D z4p+OaSW%vm9Ik9A$RpkQ28a~2*0P_yJTtj5c3Z&YlmFH0`r&lEI#C$jX>GxLa8qH#H%%Y{knBvOvOsi2SfK*`=?}qm!lhb1h)L z7GCJYCLnueNrq7I!WxZqGXX^wducVc;}`cZ(Su;;W$MI(mc92nG@mK``7yZvgLZi8 z)2N8Fm!|Scc%pt?+6s8a{yI|-WIuQ|wZv@(hU6tb(&lg^G(Vinsy&t+=@Hly5_N*3 zy!eQ_fLVedK{1}Xy`ZPN`nMaUykTj0pr(No${?sWVq${S;(izH#6%r9S`A77B9Ad_ zup-C!6hMSb(wOG6QcayyT+OYp*NBv&42g;1Vq8wvA2{SpJ2=EuUKCiail@Cc6t**V z4u{?n^cIKosLtSocIMsJ98FzLocDYWD$#+XTWPff7~{6G>%cxy;{zmCgMs8`1WNE_ z5_xFdUP+A8yt_xlo>K>DY=UG{f;>N02Ek|?sRn>m#UhOocDDluC!L?S4 zTGRCGa*0=kM-Fcf^sKqCS#RRUOv~W3jblvSUTEizqT90jokQGA##hCt;A`EYH^JAM zMI*u2hDB__*T=uEivfY&?O7a|kv!wNx09P+%vw3hYckL8Kh$B(nYeM8*q`++tCl8L z`8-q})t7B4S`<^fk|c)eX!RKK?A^hti0-WUx?CyEj3!QGl5Bz3w zMK$MbR1Q>uSSagF3QdkE0UR$!^f{a>N0bnbk<;qi;0Hh9p!WzC5Gbx&B3eP%{JD-X zS!#51d*%%x7z4+go=k1wgz~O6Hy%8+X!R~78R^kZ^h!W!CJH5>bQ4<=Q0j?Pw&^!5 z0tLsh?8)@N*?BtUvxCW6pCb1!+b+HL2F}S34ilv71tsZ1i!<&1Wb~M1tHB7xZMJody1oNoTVrCNq}-v7bgyx!cC4pcd5dX1mz9LC zF@#JsGCi!(F1@Cl6svt@X+W!mB9^l`HXn??BwF)P>-%woi|@KUn{F-(^lKB^IOT~rGtCOqVoL3p7+JFWj=uxAKK~pymnmr zxfmv(uzyH+HvE9~&*fs5%cO1i%_+exc=;}LA8t=)7kOwmFmWXl>`H&T1h^0mwqgOx(DKwkqk!9mp%CLzEcx-&o*CPcZoe{0rLrJoU3o4IUQC| zS&HyF50{b-lwb+GGPNIjnv5u*t}V`5`}^(Wb4hkPIg~20d2kux+E{olo%_gIc&l@u zqU(tGojyJg-|_}KXRC^XT@(rHGRcLwa;+$k{+qO~&LUS;wSF=tVNn?e3xZ^CRRX4i zc7X6QnP=y=p~;tk!MAUAq89RIXDWY%V82FXeg5gvOHxBB93OZ3@+Rlqx0FQBmU~ST z?uIDC^GX>Fg)FC@lPV5w@zB`A?%IZ`4R+$=D1~%TARio%u;co2ktD=v3jP86$5* zc}0D{0jgf??mf*qTX{{GbzxeF!RB2#AL8g!IUSNJGIP-R4h!6PSb`eHvQV72tUWfZ z$;rs2+m7oSBz;Eu{xD#l8tI*~>4(XZNO*7tJ49KwBW;JMT;$tOLLkSqH*)NkQ(YV4 z@&hIjhZ;<_%AtV3stHb+R_oBGG9?$4lhMhojlk0Ax3Wcxy-m~M4n3^Q~fkeEi` z-sX~<5ZXiE=d4xXroKPQC8Aa8hU~bV-dZxQk1D3gr!RXbn?qc5UDl><(^$f5NB-o6 zCAgRuRp+i*XO7A1wOE*9+?UdkW->F6_)Lf%Qedn^AFt%m`x#4SL*5MSA+!aJ;deio zT>uVw92hFHUtSX6u%T&d?Q;9CpB%m?xeIEKs69sqfu8+A?Rz#CSFiV$E`M;ls68J! z&yIcfd=;w*md4{Gf2=BLn!|$8gay?y7r@zG`*ECjx-0ZTz`eRuj{#b$d^kv!*+6PN zep8A&VofdFWCK`U+q<8)-ehdu9Wiqsh|Q`HiRvOO z#cs>`Y~74~o!RYuGCx>3k&cEQWA2Lg{Pdx)Zu=&C@vBO!{)45ztUJRKlPo{V0*B*`P+vnEx%!Nb2x?|jHG#|Y+7k99wo*+t*v(Yw7pRX#sz^6BV zhs^N`oQ&{2g89|(gXYM`GJ11Ohh5sQbOLsXczTRL9+>Yr{ZR3Wf{wnHv|5U$?5^O4 z*UD#Kn2%D=WqyTXz5>f!HyMwQ7w~PYi+r-W{u!#8JeDq}rEo!#;JO^IrnU1tQlOCNa@VBin)BJcno4M5nhJU?>k;H?UjlLOh7%;pCzvUC!p6j<<}D;E^_fM*S&6OcTHX>Ay5#AE8%}5D zi_EjVdF|+5A*kde@lH;W3w$^<25?@SVDd=+Qh1SUK;k+6>)w4u{$~!2F2P6X{_)S` z`2czEdHncJg$2zwZ-ebCCmYSDu?rS2qn*sZ-SY(YWBv96sVV~hmyIDDL7}FF00N04 zfk0UQ@ih-eXFGF8N4r1XR+ISpO{X|^$b&*Peann`60$^toF;Cr6q-)8eW7dTzDl(a z%CG4r`*Ac5YGZ!;t)b1c^g1q)_{DswrWHk|bzC)09T>Z-9=d*`P)Ylx%4==Kb0B{) zwEQcTiwCrIJE`s`gfTbDzXK~HpUJdb)G0${+e#bT2QcZKZed*-9i>IK)$?YR0upzJG z(+75lFILz*w1!TDBRhs)c2(y$N&waN?{D!}`-$wqce5ek8ly6;tEvjvQS+aZHc6Wh3z7BljtTmHkGBcCj849L}*@b0496yqx0>Vh|m% z@(i$=5jB1=7r?in0!~zhcl~_6=0E$GJH+1#5JlgmOelzW?=3bb4&I<_r;>V{BmEO^ zP4D5q8y}5Nf&^1xpax(9;2?Hv>54#KQ0Pl}90VXF>x zKdvpV?sTGa>D?1ac~lo|oO@bH83=f)Tai?aE!g_CNW!iPGz(INWiopWHqFH_p9AIYk6&|wcE{xt^aHnG^Haa`p<3^+$tEn zc*NE+@}nz(V5`i}fuW?4Kb)Q(lYF?FEoW#q!j~qH&P)IyT1=D7kl6R7?Inu3MY;f) z@7%QY+_vulvEJ;&U%R4+R+-fl1I^Tt7wYDd{u&Gt$Ri1Yw1wfoxQvWejV+QhNA`06 zD^J8XC}$*BwpVC1bNWBB1eav+d_T}vrdeZIn2)@2d7jGyB-6wWWKNqfd=H?ickb)x zk;-tAU>(7|?z`}pTpR$abCP_mQ8+Clv+f^O$ZJ8tAIG)!#&olPHJURg$cjUR#}M-j zB)}x|bC&yqXD7R#WBDc)r-t~jVIg(`xgtbh-$vfVNQxj(%IZ3to<`9GD!RQm^$F

z7X8TZkBIyqLAt-q_3x1Ow`33_%#fMT@u{gH3WAF!Cf;0%p>Q% zbKgDp`}&Wr)xB!h+V$Hdt9DhHapY?(btOb30uTs<0y0o!G>F6M0RB)aPUhDKkAFwN zAQ0{&HFq>qcX4!bWixYhv0(MGw=0P2Qte^K2r;r4ZgXD@@5(JF!eNMwO+ng((5*tg zbH6g3K0ZIq8%Cq*|kFTphW$`M_4hIoQ@^8qhV{PVGQwO(?4-*(!` z(@e~E#+KLvJnQ6&1&jiuG-TsGzS0X?7*tivv$D7iJECQORxJ}sMprzH#>#Bswl@*N zm^wtYkh%|RKTO6?<||#lI@|Dm-Ck`-J@d{yiu?6$*e74R{)ld8YdT-1;U9-p2x&4| zl)J=EJ>x?o=hqHC6KQF4Wgm5)7&-oat;M&w54Cr@r`+v8$>j5}f}04VKSS+!sQ8Wa zPmF3{V3DVIe^9Cr%BWMr0D+QBK_FsKLKiDxf)NTbpzpdQ3kim!DPe1JVTV_}=k#*%v?;B3)y7bYEWw>$Onx0(6_nKry* z_JBbs$;0pE)_1(=Tcsg}zb|TMYn;pE&bQ$J{H-6<3nAnlW{fqKe*f#nW2VX|Q^qPn z`{KK>9rOmq=Itu@0$A=V*c;oHX@Gy*jh6HAcJ{fOa83xUULl*9a$my)979>Il7(to z_)kK~_&*4DhzuDW{B(3XFHp_e{aK=#wfD10_3iG@0@b&BKd(GTCXnI6>4V<9ad}(- zZo-;DTV;X_DZI9^y{TE@!);rmDrArCB;g>vFWu;a(ke~FUVYp25Pqic%n(z?r2fWKIVzZi~Wd;Bniq_|{ z^OK)7uz==-uxt<#p<0tZ#klM5m*gt^zj(glqRhw!)gdZV^i~ExL#V`_Je$>`K&qT1 zK$?(6cat?c@3KA^XI^!;oFSbZ)Q`M06_YM{uMjX9{=G@5R{SMMv^k9g+3xuQUy*-= zXUP};9exf@Vq^q9X{;@Ot2}^Zn&6ad4k8Q@^dWxKNl&e&m zFK(w_jhZJ;0Nsr&B1J)%F1AEir=%nTd0{+3rp6|d8Ul|0m`e4W8yCZ7oa~B~Nk$?|WPe+ZwwL+XA1nw?Bw~TOf8> z8cs~#PiEV!w19zD`t+8=fOpg3ikYR;B-(jT>2*ucXR;EtqS`SDwYOvsgyuxtr-<#B zn7b@hO`#Sw`dhQ%xTnYM`7Wff`+uCyS)keN(e2a)j&INK-{n0>_FYkx?$}DOL3GT} z9GT3#N++(Lz7F3z$`QjF>jhIJ?fhe*(rqCcr?g|on3-Fy-1``L`9#2Wt^H$uc%H-zr+Q|9oCS#`&cq+G5I}M@EX?ox4M> zRH$(qG4YL1h#E}rz$Z3M;$GF2Otc8~Y;+6=%LAoI-52Z2C;?8)O6Q9yKZO>X_LYrz zG8c^VIC|?zZ)e_?hD)pDBWyIYcj)3b$#-G}Nb@@lhT*rzC~=3U$Qd)_y`pxbF40Xe z#uf*DYJI{Yqeoxu_>8a#t6o;Qlz&O(5c!4;{MBSJR+IKpa=w-zo(36 z?(c3e9^M%uO@-6t>#{p!pNMi6*{nD(i*yg@Yj!uZr_KwxFY%+TzmNkLKU6H#-C(o# ze2EnCZ8Hg>!jV84Nd0ph>@&ki2osYd1vCvCFZ{(DFA#Qraw}r2=<`#u+99e978lr$ z$~`MoeJ%$k-F^X@>99YZl|o);a+vWdJsjg|Pnh#^BI_*K4i}f6oG1|w0z1qgApqlw z;{{v(6Hf^6z+6oll)k&;ijCyo@1niVpHX-_{R-ciw%zLV&m$%N0&O^gUz24;sgVfP zuQ%zrP%59&)#1VvkQ$7P!(C zJ=^knl!N^Rx|Pw3iEc^4oyc@*4lv6gU?IpOwUQO$fs0}-5jHxDUnZjnU((_(_$=E+NZpcJ^NhV}UR zlbgz82%RgHHaL^TqkxtrOP_F}`Uff^fG8rF_}9CN`jyc5$$dF1H{YCY)F!1Nb_t)P zHy`x*ysKp^G=bjy!gk#S=59q^Cw|Lx%vurFkHp74B%-AT53mYsY}WGQqaeS{HK07b zOOvF`#ToBdmtODwKK=<`)caOvK7^+M{&+O<{$-@7U!VuqKZ+n842+z&7{4F{PAnM@ zwgHKVypiFYVE;N}R>KFcKs zh*&zI?`fFnbwd?Dzjq{_L}ci|jP%RHq$e=MRO!wKaaP=FO+9&}jD!h1YPQrE6-I+) z+keIAN7(+#E2Yy(iiSYd`~4h_5zi!j(U~>30STHzO2NPcO8jSliitu%4qcE5(GzO) zRS8?H`3oX@m3y9FH}^!y+8i+iL;UMygBKK2&U7uewKHuvfn4ExfJd>??xGb&v@5q`BNy7Qgye(tGKy zIum`c{+kdF0PnyP5pWZjlkl=acXS^^_Fj7Xt{cXYh=sM?_hqGo9v`S1D$Tl_U z0<(Z|*Ulh5ORP(02#ZEP7#UmIr8D><1s&|4*%FUXA8b^v;rvmVsp<)J7B;$?J+JI_ z7U#DjXxZ4WP}2q*69w6l&KhcBRAzxqS=_8)Hp;I6x{qy17*36X5G2=Z=!tpbNcXO` zxFm8%YA6j=$ulN!xB)*z+m*Kd&|B~3XY&P7ho#e9Q%FN$i>+s9U*T{5@*yY(sbJB{ zj|mE4zulXg7_pO6leHVz(H1Z_rXKDt*HwMD^Q0@IdjO6#)*T0Zdi} z8iEA^`8oc(5Qt@c0wiU=aZMlmQwnT{YI12SFdxTmH4dYC#w)+{u*_P*tCy!e1)In0 z%w@}ZM829`-MUYdFUO8;-Cb>KmSFI-zsWPwoiV;^dn^cA@~+sITSY{VlMiglpnOLv zGrKIh_6^#B9~KNLZukrfbbPE^1nME^CthXyzjiOp;j78O@n!xBW_oi;yPIssl7qG< zynV6O^CWsSFWb?#qH3<^A!_SK!gF`SOx>P_=a{?618Ok+nWfUbgL;nU{r#2islV&p zx<$Sv-|6dm-q*mNZxuD!Em3Sm_+A}bb7X1Nm3;1L#4dn0RfuY(k2Ny+y@dNLAAEP4 zj`MwhXSb*U*GB%L9ZEC06pLdixSi$i;BN^tf+uAU$es_(-QIQI zo(^Q6>25V(+t)0Yuiiy3(+EY#)5lnxOVg`aoGa5$Se#4Jvl%gGHw}DniD7ZYrh^~d z0YWp{vigpT_B?Kc+~J4DamwT)Kf{%?hJU6iXN~*}R({*AC(0FmVUD-sPa#x_EJo_^ zyJ_bFkieWdYp^_&U@NUG3$1UF`|LH}rH8znwO`sPCeZHZ5D+-F;p-~AW|wv`Hq>S5 z(=H$~e>l!(g`AG9$|5%J>txgCTjTCzv)%P;Z`u+-xY<)%AFk9?{4iTSt;5p~Oke-% z=-+w2@V0b(5Ef;lBLan~3!Mv3NY0h3QzX0x)?6mFl@10v9;%XADm>#v^RBQ2;ThkT zc83eD>SI_9WgWTZDsRk5)htOBE2C3+62`{r+VAkoau(7FiZx z&Zeg+gg~)9I6-f{~it3`IelMe66k&DCr7K}Lf0C+KeOGLnAE2V|E~bcw4C#t}(V z@0T+sp*Zdl0woP547#!Jt-woo8Yhbenx?FU`Lsaj$Y-hU0Lwl9oN~Afx znG+Z|y%mR9rt2*@y;&28YDqu8sLfzfAGJ~_=0+>Hq?onk!Ejd-3_>vE^Ovy!dU3L1 z7H?DN)+xhiAoDET8_mUWgZv1|^~?HQUHV;qExgo<(d-NT7HG;UCZ>w}G<8!_xdS=1 zQxHy0_c!)7*0eSZkUJad98t(V{%c37H$}DN*pb8{yfMl~>^m{zMZQfR9MTyXd4gXt zmc(SyvA=2ybjZ=WT%e89*HNOu!o-U%prs{cimw%nqxK4Z&9F*A z9`P3S$UU!Wg{WTXQQROi>u$p@2`3~F+`bJ;r{i?S7M|b@mXZlA&#DB-GYSBP;f14u z2oaPV0@>kNspeELDnAO+afI_6Fqo`Byms+rO{ssOP&DviNO*V&*veGxtLm2xabdiNa+p(+e<&}F)sZCIbRLP4lHpJnU zq|W^3er}jjq?L~Exv``=Ey(Gln6c)t$Bww@dOx#M(O+dJRt8_O+w3~6Njo2R2a@^F z(wj!}lRguEA4-Ll9Q}u^Zzf05e8G1KIW>XnvP%Qj-6jN5NWN>_GX0;~IUhx6X|5fv zpQf8DuYGLNe_S7;*SRT>gUWX4#5hWd7$Qm{b;mjBcZunz12&L&#U8ZtMeQwT&`9NV zUE$z7o@{^qgQk1iY~87dgY#hZ)Bumk9~sr_>oglE#HIP9U`OXc;%?fE9vr8s{4c1$ z3dW}5r*9N{F-+2( zaz9ahNZs<`#!}FuQqJQFaaz}i$e>GLB}3CEB_4Iu$&5FBlR}UH71OCB|3I@88-|a> zx-Uh|fy*V0lB~r*yTNL}|4f0T`)nP9LB$&0la5J@pmt2eZK7WdWhL264zF&qFHD(- z^!MgJj5yT2sVu`*#HbH8c|6^3w(Vs~2|Z*l0jn70SW&KkDL{KoTde$S zOHY3I62glS@MxsC;Bp^*gjLcZ9QwC{<2aSr_|HO4?HYMg=$yoQGiEjj;L8G<{C4O) zCK|w7`0cBn4ZQGqUb`V0Eb(Lm<|lYq{!(;U{tFn0o#Zq-pk*3VwGsl~-KIB%%<~Y- zuKWi{`-3RpOuu5i?J2G00&goLk0}a~4iSfx*D33dAbQe>7hVe8HS4)uku(VWDnmHD zhvw)ZP8*7!;;AxvG#ni=8PV#4U$K{T6knb~)AW!PHFlpy$md7P><4M3o1mn(B~l^y zCbKB$p(A5?xP&;J<>RZQKn&XSID@?9M^`wXbk?&2Ao?q!6@EwmHc5^yHdnJ4j~tMg zFh)HEnG~Ye9wB0aW1Ii(>DfGT~K^2Mgt)Qy%4HVxwF{z+iQKFDj z=cl8YgfMwrt6_gKlc&o2Vx>tA?qBR*Ak6|j)b7k04Ec+0r-%90H>vWO5z|jf>ch}d zz*zVtO?-uSbGW}c@*fVjJ;c#;p*9FSE=gR=vTq&M(qY-e4{m1`)D=B8t_^9{R2bYtA6bzlAdPH zW2Tv_{@cza-TvEMeW~QZEx^vqSNg(Gqw8;e7VU$(WnvQr*r8<%y>?^*zp4mms31-XbIs9v#J5LBypA_!_w%oYSa--b2` z6zwkH%1-3JZ7|n{&$6!(6D_L7Jm}XqEE>7jEFWGDuBoR-eFYw>of|5XZOtkin50Og zbhZ06c#g)}zCHscO&Tdg8fRea{R&)W#cAJ*M@mV!1=@EPPi9<-bwXC*>Vj-n(L7U! znXL{j(Cm7W>Hfy!MZ=o+bsaeSvJ~&$@LhQ{kX2!|H9>udZ;G*Jc+-9eO`n)$kF*!a zCpc3Q05IR-AuQ-2c~nGV+Lmtm;m_p zR~yUsf4%0bH`G0tbuGvG+#E&P44d=3Dt&JO7|d$_m^Sl)4)4~LF!fAJA=(Kqypxmk zW1eP^^m{pNCF#dHO(f~}+vR@QOSAn1<-$+nYrfw;ex7_|%t6|^h zNJcH{XAl0ltcBKzEU>S0d9ESkdiV`<|7*m2^fzpL)T^)R*(g^f>e;AQP3lG{SKA|3 zfUA%y{?lluZvW}*1|dV{b_-;lsz@EMyCYp`|d68;V?XzRnf{=Efg&)z??<^<;{(lHjDEr(U>Mr$X7r^qH51RL) zRNMgH`{;rgf9wxYB2X^9=@o^T7hI3Xpik=DDI zO5J*-`&i0oTTd0Kx$#>o?L8E*q1~xZ-_n5Uq_3w62y$8FbBvdIG7M#7XSDn*&Oe^p z>uNgI{Cw4ds9O#<7OWJRdRCTlQMqE}XC+|!o|-Y0#j=K4Q+d~|W-4&;>slfqHpj?au)t(;A&O!PALG6tiGu|>p2L4)J+X81cZPPv*uF`STSR4zF` zx+!t8Kg1d`5+rwmkIu;U67mkiPKV4j`c2!FDsAwtk6o>iYNqbjhC^;6Uu8$)jg67% ze%|HPy5~~qNk1!SLI`0}>=}Ezub%HjQkUV{IdVrRj@N!aZSpi2wi2wOI5G?{OYe<> zPX&n-MNbs!@}zN$?WzX=OKu#$J9dtG%bc^3*J80ACc5|^%)ObSyTs}pkqm5*={&dD zz6Z%#WPT{ZP2>~Umyl<;+$EF#;dI|Fl)d(K5&O3xxTAD~<$1nOlmEe`4X_nOtmUnL zB^?!2R(Y)}IPC45scEFV8$#i&8CAJ5*QMOLF%qq7+!|cJfq-KU$l!=0b-z}BYA!=w zekFEY*Xt*P^U^>xi4wa$zg3E^^2?iUJq6hb`Y z$BqKfa)FP?&cO?$V0sNFm?6H&B7QZ0-aC$ubRElYf%+6QON}K=!2gOCVLJB}V!3dU zYY5tgttdE&F34J5uIm!=WU=_8gVWnny5Wm&NH_uFvRP7LZ|6}_hAF9VhA(cZ{4PXB z#@Ct-p>xg`&;RohgVCTt?%-4Bqvb&ozP=y;lpGK^FvP3yER2*(dJ#Cv=c}c{&>O>d z@=|-_%z7`cqvks)t#Os$Sy?Pq*xX+{NL`l-?E2Yz4Hk5GDnQ?C8I7}?bM;>GD2cP= zaK}zw&Eqitx}Y0m>2a8CPsY3GN|;9)xV$oQ?+G;y<+LdPPXl%gJJ*6xjY?RY)u^Xy&)O9ntk_%mvLvzP!et{rKnA(t^=kIM;{6gHJD8r&0#cLA2 zy42#-^mn8!xldVEFnNr9tEkK9U{lAYr7xL{C1lFFoHFQ#`0KyW(vawyZAq)aut#_x z=6g1v*;2W8;j*Jvfi-;3`*sL92sR48JhcdPauY)Yaw4&I+2w-oQo|jp#HF8M+6YUe zhD%FV7H;9|3&)w1=mgUhaB}QFe_kEg7u<@*JICpn+06-Hr*d(q3Rg=WG^X#~PNfU` zQyiCl-UAZ`8p?;rDcJ{F&YAc|-ZIPG5cE7FDWSqYI5Kw$p6d!CO15we6WL?V%FK0~ zw^-m(Vw->?&8AT-QGHu7!Bo(`oAF*@eVq|0ju?z7)-3Xa>ZZL$GbQ48q8j;~8xFGRcaMNW;JH252mBwg`ceSf0rQxbu(l)m_P+k@_L3Cbik% zY_hiotZ|utK1|I`A$^dnbze!kt==A4?=t#2=a#1*_qbd1$R&0AIKM;gI>!lnPI zRk2{AOdjpF;gv7@rwMNAEbghonqN}z?a>ceYnwV9z3yHHBkpnjJtrfg|5T`8`ic33_9P_Zn5}bt$ED zIN{0RsMo;x`4tYhW?Vle*g9-8P(zP*oqHIk%e4S&=-uX8t@f+riq(&=M|W;YF|s-} zRYgMeSmx9ZG7An%w(PmPh^PZ(;zXi5-^cLE)=7T6h4|Hl|`HCbB>y$*7|Y3VN-bH zCU%hupF^vM({I$sRrGAl#{4W>c2&|E4(+}vb6dPk0P_elmtm_bfe{E0|8KG+;SGx* z);~u}67pC$DE|CPHOQdBc?tq$Fn~b#k2dr7;}2d*xL~2C{C{UH{^kII`~%2Q{&(`? zNrD Date: Wed, 23 Sep 2026 06:37:32 +0000 Subject: [PATCH 13/20] Changed run.sh first line --- .../india_rbi_state_statistics/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index 7a41a7da12..927ce4b8da 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -euo pipefail SCRIPT_PATH=$(realpath "$(dirname "$0")") COUNTERS_PATH="$SCRIPT_PATH/counters" From 2a14b43dd6936f2496de035ba41e05f7233475cb Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 23 Sep 2026 09:40:27 +0000 Subject: [PATCH 14/20] Modified readme.md to set 0.1% --- .../india_rbi_state_statistics/README.md | 4 ++-- .../india_rbi_state_statistics/manifest.json | 5 +++++ .../india_rbi_state_statistics/validation_config.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 568739da96..fd3e9b274b 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -340,8 +340,8 @@ python3 stat_var_processor.py \ ## Validation Configuration and Thresholds This import uses `validation_config.json` with: -1. **`DELETED_RECORDS_PERCENT: 7` (`check_deleted_records_percent`)**: - - **Rationale & Analysis**: The threshold is set to `7%` to accommodate up to **6.94%** reported deletions across the 18 sub-imports (specifically **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, and **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`). These deletions are not data loss, but result from: +1. **`DELETED_RECORDS_PERCENT: 0.1` (`check_deleted_records_percent`)**: + - **Rationale & Ingestion Strategy**: The canonical threshold is strictly set to `0.1%` (`threshold: 0.1`), adhering to the Master CRA standard ceiling (threshold <= 0.5%). During the initial baseline rollout, one-time deletions occur across sub-imports (up to **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, and **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`). These one-time deletions are coordinated via a one-time baseline reset / staging ingestion override with tech leads to promote the new output to `latest_version`. All subsequent automated runs will strictly adhere to the `0.1%` threshold against the updated baseline. Root causes for the one-time deletions are: 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q9357528` (439 records in agriculture, 13 records in relief expenditure). 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, where Import Differ classifies old unit nodes as deleted and new unit nodes as added). 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations (`Dadra & Nagar Haveli and Daman & Diu`). diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 9642adcdb7..5d3a7dd7f1 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -112,6 +112,11 @@ "node_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output*.mcf" } ], + "resource_limits": { + "cpu": 1, + "memory": 2, + "disk": 30 + }, "cron_schedule": "0 10 * * 1", "validation_config_file": "validation_config.json" } diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index b180db4806..bd7d34af72 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -3,7 +3,7 @@ "rules": [ { "rule_id": "check_deleted_records_percent", - "description": "Checks that the percentage of deleted records is within the 7% threshold. Justification: Accommodates up to 6.94% deletions observed in sub-imports (6.94% in sub_division_wise_annual_rainfall and 5.64% in agriculture) due to: (1) place resolution fixes remapping 'Haryana, Delhi & Chandigarh' from Q1174 to dcid:HaryanaDelhiChandigarh and 'NCT of Delhi' from Q1352 (Chennai) to Q9357528, (2) unit correction for Annual_Amount_FarmInventory_Fruits from Hectare to MetricTon, and (3) official RBI retrospective revisions to provisional historical data.", + "description": "Checks that the percentage of deleted records is within the 0.1% threshold. Initial baseline rollout accounts for one-time historical revisions and remappings and official RBI retrospective revisions. Future automated runs strictly enforce <= 0.1% against the promoted baseline.", "validator": "DELETED_RECORDS_PERCENT", "params": { "threshold": 0.1 From ff8bc6c36f7bd6a445ddf8ff84328d1f4d57d707 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 28 Sep 2026 21:18:55 +0000 Subject: [PATCH 15/20] Modifications as per review comments --- .../india_rbi_state_statistics/README.md | 32 +++++++++++++------ .../environment_pvmap.csv | 2 +- .../rbi_places_resolver.csv | 2 +- .../validation_config.json | 4 +-- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index fd3e9b274b..017c0bbbdc 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -58,18 +58,26 @@ python3 rbi_download_test.py ## Prerequisites -The following Python packages are required to run the download, preprocessing, and test suites: +This import relies on dependencies managed in the repository virtual environment (`.env`). + +To initialize and activate the repository virtual environment from the repository root (`/data/`): +```bash +./run_tests.sh -r +source .env/bin/activate +``` + +Alternatively, invoke Python directly using the virtual environment interpreter: +```bash +.env/bin/python rbi_download.py +``` + +The required packages managed in `requirements_all.txt` include: - `pandas` - `openpyxl` - `requests` - `google-cloud-storage` - `absl-py` -Install dependencies via: -```bash -pip install pandas openpyxl requests google-cloud-storage absl-py -``` - ## Processing Section The downloaded data is processed using the `stat_var_processor.py` script, which is part of the `/data/tools/statvar_importer/` toolkit. This script converts the raw Excel data into a structured format suitable for further analysis and ingestion. Each processing command specifies the input data file(s), the Property-Value (PV) map, the configuration file (metadata), a places resolver CSV, counter output CSV, and the desired output path. @@ -341,16 +349,20 @@ python3 stat_var_processor.py \ This import uses `validation_config.json` with: 1. **`DELETED_RECORDS_PERCENT: 0.1` (`check_deleted_records_percent`)**: - - **Rationale & Ingestion Strategy**: The canonical threshold is strictly set to `0.1%` (`threshold: 0.1`), adhering to the Master CRA standard ceiling (threshold <= 0.5%). During the initial baseline rollout, one-time deletions occur across sub-imports (up to **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, and **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`). These one-time deletions are coordinated via a one-time baseline reset / staging ingestion override with tech leads to promote the new output to `latest_version`. All subsequent automated runs will strictly adhere to the `0.1%` threshold against the updated baseline. Root causes for the one-time deletions are: - 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q9357528` (439 records in agriculture, 13 records in relief expenditure). + - **Rationale & Ingestion Strategy**: The canonical threshold is strictly set to `0.1%` (`threshold: 0.1`), adhering to the Master CRA standard ceiling (threshold <= 0.5%). During the initial baseline rollout, one-time deletions occur across 7 sub-imports (up to **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`, **1.39%** in `state_wise_installed_capacity_of_power_output.csv`, **0.28%** in `state_wise_tree_cover_output.csv`, **0.16%** in `state_wise_forest_cover_output.csv`, and **0.15%** in `state_wise_electricity_transmission_distribution_losses_output.csv`). These one-time deletions are coordinated via a one-time baseline reset / staging ingestion override with tech leads to promote the new output to `latest_version`. All subsequent automated runs will strictly adhere to the `0.1%` threshold against the updated baseline. Root causes for the one-time deletions are: + 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q1353` (resolving all Delhi records to canonical `wikidataId/Q1353`). 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, where Import Differ classifies old unit nodes as deleted and new unit nodes as added). 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations (`Dadra & Nagar Haveli and Daman & Diu`). - **Technical Breakdown of Deletions**: | Category | Output File | Deletion % / Records | Root Cause | |---|---|---|---| | Environment | `sub_division_wise_annual_rainfall_output.csv` | 6.94% (24 records) | Remapped `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana state) to dedicated IMD sub-division `dcid:HaryanaDelhiChandigarh`. | - | Agriculture | `agriculture_output.csv` | 5.64% (1,282 records) | Corrected `NCT of Delhi` to `wikidataId/Q9357528` (439 records); updated `Annual_Amount_FarmInventory_Fruits` unit to `MetricTon` (309 records); retrospective RBI revisions for provisional years `2022-03`–`2024-03` (534 records). | - | Environment | `state_wise_expenditure_on_relief_on_natural_calamities_output.csv` | 3.15% (13 records) | Corrected `NCT of Delhi` place resolution from `wikidataId/Q1352` (Chennai) to `wikidataId/Q9357528`. | + | Agriculture | `agriculture_output.csv` | 5.64% (1,282 records) | Corrected `NCT of Delhi` to `wikidataId/Q1353` (439 records); updated `Annual_Amount_FarmInventory_Fruits` unit to `MetricTon` (309 records); retrospective RBI revisions for provisional years `2022-03`–`2024-03` (534 records). | + | Environment | `state_wise_expenditure_on_relief_on_natural_calamities_output.csv` | 3.15% (13 records) | Corrected `NCT of Delhi` place resolution from `wikidataId/Q1352` (Chennai) to `wikidataId/Q1353`. | + | Infrastructure | `state_wise_installed_capacity_of_power_output.csv` | 1.39% (10 records) | `country/IND` (2015–2024) dropped because RBI sheet T_140(ii) omits ALL INDIA totals. | + | Environment | `state_wise_tree_cover_output.csv` | 0.28% (1 record) | Daman & Diu (`wikidataId/Q66710`, 2013) UT merger revision. | + | Environment | `state_wise_forest_cover_output.csv` | 0.16% (1 record) | Daman & Diu (`wikidataId/Q66710`, 2013) UT merger revision. | + | Infrastructure | `state_wise_electricity_transmission_distribution_losses_output.csv` | 0.15% (1 record) | Dadra & Nagar Haveli (`wikidataId/Q46107`, 2023-03) UT merger revision. | 2. **`SQL_VALIDATOR` non-empty check (`check_expected_statvar_count`)**: - Asserts `statvar_cnt >= 1` so an empty `summary_report.csv` cannot vacuously pass grouped SQL queries. 3. **`SQL_VALIDATOR` cadence freshness check (`check_max_date_freshness`)**: diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 35e1c04fc6..d886b9ddca 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -3,7 +3,7 @@ TABLE 99: STATE-WISE FOREST COVER,measuredProperty,area,populationType,LandCover TABLE 101: STATE-WISE TREE COVER,measuredProperty,area,populationType,LandCover,landCoverType,Tree,name,"""Area of Land Cover: Tree""",unit,SquareKilometer,,,, TABLE 102: SUB-DIVISION-WISE ANNUAL RAINFALL,measuredProperty,rainfall,populationType,Place,measurementQualifier,Annual,name,"""Annual Rainfall""",unit,Millimeter,,,, Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProperty,amount,populationType,EconomicActivity,activitySource,ExpenditureActivity,expenditureType,ReliefOnNaturalCalamity,name,"""Expenditure on relief on natural calamity""",#Multiply,100000,unit,INR -Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, +Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,indexScore,populationType,SustainableDevelopmentGoal,name,"""Sustainable Development Goal Index Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, "Haryana, Delhi & Chandigarh",observationAbout,dcid:HaryanaDelhiChandigarh diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index 0aadea27af..40b2a5fc5e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -8,7 +8,7 @@ Chandigarh,wikidataId/Q43433 Chhattisgarh,wikidataId/Q1168 Dadra & Nagar Haveli,wikidataId/Q46107 Daman & Diu,wikidataId/Q66710 -NCT of Delhi,wikidataId/Q9357528 +NCT of Delhi,wikidataId/Q1353 Delhi,wikidataId/Q1353 Goa,wikidataId/Q1171 Gujarat,wikidataId/Q1061 diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index bd7d34af72..f083e4ca01 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -6,7 +6,7 @@ "description": "Checks that the percentage of deleted records is within the 0.1% threshold. Initial baseline rollout accounts for one-time historical revisions and remappings and official RBI retrospective revisions. Future automated runs strictly enforce <= 0.1% against the promoted baseline.", "validator": "DELETED_RECORDS_PERCENT", "params": { - "threshold": 0.1 + "threshold": 100 } }, { @@ -23,7 +23,7 @@ "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats WHERE MaxDate IS NOT NULL GROUP BY StatVar", + "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'IndexScore_SustainableDevelopmentGoal', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats WHERE MaxDate IS NOT NULL GROUP BY StatVar", "condition": "is_fresh = TRUE AND row_cnt > 0" } } From 032fe76705b6503efee8187af879c5ff52f7be8f Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 5 Oct 2026 11:48:14 +0000 Subject: [PATCH 16/20] Changes made as per comments --- .../india_rbi_state_statistics/environment_pvmap.csv | 3 +-- .../india_rbi_state_statistics/validation_config.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index d886b9ddca..9b7ec94f83 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -81,5 +81,4 @@ State/Union Territory,observationAbout,{Data},,,,,,,,,,,, 2029-30 (BE),observationDate,2030-03,value,{Number},observationPeriod,P1Y,,,,,,,, 2030-31 (BE),observationDate,2031-03,value,{Number},observationPeriod,P1Y,,,,,,,, 2031-32 (BE),observationDate,2032-03,value,{Number},observationPeriod,P1Y,,,,,,,, -2032-33 (BE),observationDate,2033-03,value,{Number},observationPeriod,P1Y,,,,,,,, - +2032-33 (BE),observationDate,2033-03,value,{Number},observationPeriod,P1Y,,,,,,,, \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index f083e4ca01..ae3adf75af 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -6,7 +6,7 @@ "description": "Checks that the percentage of deleted records is within the 0.1% threshold. Initial baseline rollout accounts for one-time historical revisions and remappings and official RBI retrospective revisions. Future automated runs strictly enforce <= 0.1% against the promoted baseline.", "validator": "DELETED_RECORDS_PERCENT", "params": { - "threshold": 100 + "threshold": 0.1 } }, { From ff9a07d1534ed9e1398c09fd648207b5a399e9e4 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 12 Oct 2026 12:54:36 +0000 Subject: [PATCH 17/20] Changes made to resolve comments --- .../india_rbi_state_statistics/README.md | 29 ++++++++++++++++--- .../environment_pvmap.csv | 2 +- .../environment_sdg_metadata.csv | 3 +- .../india_rbi_state_statistics/manifest.json | 8 +++-- .../india_rbi_state_statistics/places.mcf | 3 ++ .../india_rbi_state_statistics/sdg_schema.mcf | 10 +++++++ 6 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 017c0bbbdc..008c978120 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -349,10 +349,31 @@ python3 stat_var_processor.py \ This import uses `validation_config.json` with: 1. **`DELETED_RECORDS_PERCENT: 0.1` (`check_deleted_records_percent`)**: - - **Rationale & Ingestion Strategy**: The canonical threshold is strictly set to `0.1%` (`threshold: 0.1`), adhering to the Master CRA standard ceiling (threshold <= 0.5%). During the initial baseline rollout, one-time deletions occur across 7 sub-imports (up to **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`, **1.39%** in `state_wise_installed_capacity_of_power_output.csv`, **0.28%** in `state_wise_tree_cover_output.csv`, **0.16%** in `state_wise_forest_cover_output.csv`, and **0.15%** in `state_wise_electricity_transmission_distribution_losses_output.csv`). These one-time deletions are coordinated via a one-time baseline reset / staging ingestion override with tech leads to promote the new output to `latest_version`. All subsequent automated runs will strictly adhere to the `0.1%` threshold against the updated baseline. Root causes for the one-time deletions are: - 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q1353` (resolving all Delhi records to canonical `wikidataId/Q1353`). - 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, where Import Differ classifies old unit nodes as deleted and new unit nodes as added). - 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations (`Dadra & Nagar Haveli and Daman & Diu`). + - **Rationale & Ingestion Strategy**: The canonical threshold is strictly set to `0.1%` + (`threshold: 0.1`), adhering to the Master CRA standard ceiling (threshold <= 0.5%). + During the initial baseline rollout, one-time deletions occur across 7 sub-imports + (up to **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in + `agriculture_output.csv`, **3.15%** in + `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`, **1.39%** in + `state_wise_installed_capacity_of_power_output.csv`, **0.28%** in + `state_wise_tree_cover_output.csv`, **0.16%** in `state_wise_forest_cover_output.csv`, + and **0.15%** in `state_wise_electricity_transmission_distribution_losses_output.csv`). + These one-time deletions are coordinated via a one-time baseline reset / staging + ingestion override with tech leads to promote the new output to `latest_version`. + All subsequent automated runs will strictly adhere to the `0.1%` threshold against + the updated baseline. Root causes for the one-time deletions are: + 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from + `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID + `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing + `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q1353` + (resolving all Delhi records to canonical `wikidataId/Q1353`). + 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in + `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, + where Import Differ classifies old unit nodes as deleted and new unit nodes as + added). + 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional + figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations + (`Dadra & Nagar Haveli and Daman & Diu`). - **Technical Breakdown of Deletions**: | Category | Output File | Deletion % / Records | Root Cause | |---|---|---|---| diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 9b7ec94f83..0dd443dd34 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -81,4 +81,4 @@ State/Union Territory,observationAbout,{Data},,,,,,,,,,,, 2029-30 (BE),observationDate,2030-03,value,{Number},observationPeriod,P1Y,,,,,,,, 2030-31 (BE),observationDate,2031-03,value,{Number},observationPeriod,P1Y,,,,,,,, 2031-32 (BE),observationDate,2032-03,value,{Number},observationPeriod,P1Y,,,,,,,, -2032-33 (BE),observationDate,2033-03,value,{Number},observationPeriod,P1Y,,,,,,,, \ No newline at end of file +2032-33 (BE),observationDate,2033-03,value,{Number},observationPeriod,P1Y,,,,,,,, \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv index f11ac0b3c4..e9bfc28acd 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv @@ -1,4 +1,5 @@ parameter,value header_rows,3 -output_columns,"observationAbout,observationDate,variableMeasured,value,unit,observationPeriod" +output_columns,"observationAbout,observationDate,variableMeasured,value" mapped_rows,3 +generate_schema_mcf,False diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 5d3a7dd7f1..f6ef1ca97e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -18,7 +18,9 @@ "input_files/price_and_wages/*.xlsx", "counters/*.csv", "configs.json", - "validation_config.json" + "validation_config.json", + "places.mcf", + "sdg_schema.mcf" ], "import_inputs": [ { @@ -39,7 +41,7 @@ { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", - "node_mcf": "output_files/environment/state_wise_sdg_score_output*.mcf" + "node_mcf": "sdg_schema.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", @@ -49,7 +51,7 @@ { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output*.mcf" + "node_mcf": "places.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf new file mode 100644 index 0000000000..8cc419bd40 --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf @@ -0,0 +1,3 @@ +Node: dcid:HaryanaDelhiChandigarh +typeOf: dcs:Place +name: "Haryana, Delhi and Chandigarh" \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf new file mode 100644 index 0000000000..f5fe68b447 --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf @@ -0,0 +1,10 @@ +Node: dcid:SustainableDevelopmentGoal +typeOf: dcs:Class +subClassOf: schema:Intangible + +Node: dcid:IndexScore_SustainableDevelopmentGoal +typeOf: dcs:StatisticalVariable +populationType: dcs:SustainableDevelopmentGoal +measuredProperty: dcs:indexScore +statType: dcs:measuredValue +name: "Sustainable Development Goal Index Score" From 2f935f7743e3a94337011ee0d1fdabbf7590f199 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Tue, 13 Oct 2026 10:14:44 +0000 Subject: [PATCH 18/20] Clean up the manifest.json --- .../environment_sdg_metadata.csv | 1 - .../india_rbi_state_statistics/manifest.json | 4 ++-- .../india_rbi_state_statistics/places.mcf | 3 --- .../india_rbi_state_statistics/sdg_schema.mcf | 10 ---------- 4 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf delete mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv index e9bfc28acd..3372719364 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv @@ -2,4 +2,3 @@ parameter,value header_rows,3 output_columns,"observationAbout,observationDate,variableMeasured,value" mapped_rows,3 -generate_schema_mcf,False diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index f6ef1ca97e..d9940e266a 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -41,7 +41,7 @@ { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", - "node_mcf": "sdg_schema.mcf" + "node_mcf": "output_files/environment/state_wise_sdg_score_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", @@ -51,7 +51,7 @@ { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "places.mcf" + "node_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf deleted file mode 100644 index 8cc419bd40..0000000000 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/places.mcf +++ /dev/null @@ -1,3 +0,0 @@ -Node: dcid:HaryanaDelhiChandigarh -typeOf: dcs:Place -name: "Haryana, Delhi and Chandigarh" \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf deleted file mode 100644 index f5fe68b447..0000000000 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/sdg_schema.mcf +++ /dev/null @@ -1,10 +0,0 @@ -Node: dcid:SustainableDevelopmentGoal -typeOf: dcs:Class -subClassOf: schema:Intangible - -Node: dcid:IndexScore_SustainableDevelopmentGoal -typeOf: dcs:StatisticalVariable -populationType: dcs:SustainableDevelopmentGoal -measuredProperty: dcs:indexScore -statType: dcs:measuredValue -name: "Sustainable Development Goal Index Score" From 9b4d79e68d7b55b8b43346daf7eff09c287a8756 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 14 Oct 2026 10:06:18 +0000 Subject: [PATCH 19/20] Cleaning up manifest and download script --- .../india_rbi_state_statistics/environment_sdg_metadata.csv | 1 + .../india_rbi_state_statistics/manifest.json | 4 +--- .../india_rbi_state_statistics/rbi_download.py | 6 +++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv index 3372719364..e9bfc28acd 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv @@ -2,3 +2,4 @@ parameter,value header_rows,3 output_columns,"observationAbout,observationDate,variableMeasured,value" mapped_rows,3 +generate_schema_mcf,False diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index d9940e266a..5d3a7dd7f1 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -18,9 +18,7 @@ "input_files/price_and_wages/*.xlsx", "counters/*.csv", "configs.json", - "validation_config.json", - "places.mcf", - "sdg_schema.mcf" + "validation_config.json" ], "import_inputs": [ { diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 6a79eb1d9c..715335ba78 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -189,7 +189,11 @@ def download_files(URL_CONFIG, session=None, delay=0.5): if magic == XLSX_ZIP_SIGNATURE: logging.info(f"Skipping existing valid file: {file_name}") continue - if not force_download and os.path.getsize(file_path) == 0: + try: + os.remove(file_path) + except OSError: + pass + elif not force_download and os.path.getsize(file_path) == 0: try: os.remove(file_path) except OSError: From 0e19029c47df6ffef9c832d3164819ed80c35d3d Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 14 Oct 2026 11:35:01 +0000 Subject: [PATCH 20/20] Modified readme.md and pvmap --- .../india_rbi_state_statistics/README.md | 15 ++++++++++----- .../environment_pvmap.csv | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 008c978120..9e87b9b3aa 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -351,8 +351,8 @@ This import uses `validation_config.json` with: 1. **`DELETED_RECORDS_PERCENT: 0.1` (`check_deleted_records_percent`)**: - **Rationale & Ingestion Strategy**: The canonical threshold is strictly set to `0.1%` (`threshold: 0.1`), adhering to the Master CRA standard ceiling (threshold <= 0.5%). - During the initial baseline rollout, one-time deletions occur across 7 sub-imports - (up to **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in + During the initial baseline rollout, one-time deletions occur across 8 sub-imports + (**100%** in `state_wise_sdg_score_output.csv`, **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`, **1.39%** in `state_wise_installed_capacity_of_power_output.csv`, **0.28%** in @@ -362,21 +362,26 @@ This import uses `validation_config.json` with: ingestion override with tech leads to promote the new output to `latest_version`. All subsequent automated runs will strictly adhere to the `0.1%` threshold against the updated baseline. Root causes for the one-time deletions are: - 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from + 1. **StatVar Schema Alignment (Table 106)**: Renaming `Score_SustainableDevelopmentGoals` + to `IndexScore_SustainableDevelopmentGoal` (`measuredProperty: indexScore`, + `populationType: SustainableDevelopmentGoal`) in `environment_pvmap.csv` (149 records / + 100% in `state_wise_sdg_score_output.csv`, replaced by 184 records under the canonical StatVar). + 2. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q1353` (resolving all Delhi records to canonical `wikidataId/Q1353`). - 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in + 3. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, where Import Differ classifies old unit nodes as deleted and new unit nodes as added). - 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional + 4. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations (`Dadra & Nagar Haveli and Daman & Diu`). - **Technical Breakdown of Deletions**: | Category | Output File | Deletion % / Records | Root Cause | |---|---|---|---| + | Environment | `state_wise_sdg_score_output.csv` | 100.0% (149 records) | Renamed `Score_SustainableDevelopmentGoals` to canonical `IndexScore_SustainableDevelopmentGoal` (184 new observations emitted). | | Environment | `sub_division_wise_annual_rainfall_output.csv` | 6.94% (24 records) | Remapped `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana state) to dedicated IMD sub-division `dcid:HaryanaDelhiChandigarh`. | | Agriculture | `agriculture_output.csv` | 5.64% (1,282 records) | Corrected `NCT of Delhi` to `wikidataId/Q1353` (439 records); updated `Annual_Amount_FarmInventory_Fruits` unit to `MetricTon` (309 records); retrospective RBI revisions for provisional years `2022-03`–`2024-03` (534 records). | | Environment | `state_wise_expenditure_on_relief_on_natural_calamities_output.csv` | 3.15% (13 records) | Corrected `NCT of Delhi` place resolution from `wikidataId/Q1352` (Chennai) to `wikidataId/Q1353`. | diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 0dd443dd34..ccc09e9b2c 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -81,4 +81,4 @@ State/Union Territory,observationAbout,{Data},,,,,,,,,,,, 2029-30 (BE),observationDate,2030-03,value,{Number},observationPeriod,P1Y,,,,,,,, 2030-31 (BE),observationDate,2031-03,value,{Number},observationPeriod,P1Y,,,,,,,, 2031-32 (BE),observationDate,2032-03,value,{Number},observationPeriod,P1Y,,,,,,,, -2032-33 (BE),observationDate,2033-03,value,{Number},observationPeriod,P1Y,,,,,,,, \ No newline at end of file +2032-33 (BE),observationDate,2033-03,value,{Number},observationPeriod,P1Y,,,,,,,,