From aa850bae0778d70fe8d08a18dfb81c6721e09c5c Mon Sep 17 00:00:00 2001 From: Ashwani Srivastav Date: Wed, 22 Jul 2026 10:35:17 +0000 Subject: [PATCH 1/5] added goldens --- .../ndap/golden_data/golden_observations.csv | 22 ++++++++++++++ .../golden_data/golden_summary_report.csv | 4 +++ statvar_imports/india_ndap/ndap/manifest.json | 3 +- .../india_ndap/ndap/validation_config.json | 29 +++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 statvar_imports/india_ndap/ndap/golden_data/golden_observations.csv create mode 100644 statvar_imports/india_ndap/ndap/golden_data/golden_summary_report.csv create mode 100644 statvar_imports/india_ndap/ndap/validation_config.json diff --git a/statvar_imports/india_ndap/ndap/golden_data/golden_observations.csv b/statvar_imports/india_ndap/ndap/golden_data/golden_observations.csv new file mode 100644 index 0000000000..d1230e903f --- /dev/null +++ b/statvar_imports/india_ndap/ndap/golden_data/golden_observations.csv @@ -0,0 +1,22 @@ +"observationAbout" +"wikidataId/Q1177" +"wikidataId/Q22424" +"wikidataId/Q1499" +"wikidataId/Q1174" +"wikidataId/Q1353" +"wikidataId/Q1437" +"wikidataId/Q1498" +"wikidataId/Q1165" +"wikidataId/Q1164" +"wikidataId/Q1356" +"wikidataId/Q1184" +"wikidataId/Q22048" +"wikidataId/Q1168" +"wikidataId/Q1188" +"wikidataId/Q1061" +"wikidataId/Q1191" +"wikidataId/Q1159" +"wikidataId/Q1185" +"wikidataId/Q1186" +"wikidataId/Q1445" +"wikidataId/Q677037" diff --git a/statvar_imports/india_ndap/ndap/golden_data/golden_summary_report.csv b/statvar_imports/india_ndap/ndap/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..2bfb6dea68 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/golden_data/golden_summary_report.csv @@ -0,0 +1,4 @@ +"MeasurementMethods","Units","ScalingFactors","NumPlaces","StatVar","observationPeriods","MinDate" +"[]","[Year]","[]","21","LifeExpectancy_Person_Male","[P4Y]","1997" +"[]","[Year]","[]","21","LifeExpectancy_Person_Female","[P4Y]","1997" +"[]","[Year]","[]","21","LifeExpectancy_Person","[P4Y]","1997" diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json index a3e926acf9..7097abff43 100644 --- a/statvar_imports/india_ndap/ndap/manifest.json +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -20,7 +20,8 @@ "source_files": [ "input_files/India_LifeExpectancy_input.csv" ], - "cron_schedule": "00 11 1,15 * *" + "cron_schedule": "00 11 1,15 * *", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/validation_config.json b/statvar_imports/india_ndap/ndap/validation_config.json new file mode 100644 index 0000000000..eb677553ee --- /dev/null +++ b/statvar_imports/india_ndap/ndap/validation_config.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "Check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output/India_LifeExpectancy.csv" + } + } + ] +} From 750588185ed55d7fa090a2d05e3db1cf0dd74ca0 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Mon, 27 Jul 2026 16:00:49 +0530 Subject: [PATCH 2/5] Rename rule_id to check_goldens_output_csv --- statvar_imports/india_ndap/ndap/validation_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/india_ndap/ndap/validation_config.json b/statvar_imports/india_ndap/ndap/validation_config.json index eb677553ee..92ba230f2f 100644 --- a/statvar_imports/india_ndap/ndap/validation_config.json +++ b/statvar_imports/india_ndap/ndap/validation_config.json @@ -17,7 +17,7 @@ } }, { - "rule_id": "Check_goldens_output_csv", + "rule_id": "check_goldens_output_csv", "description": "Verifies the generated output CSV data matches established critical golden records", "validator": "GOLDENS_CHECK", "params": { From a8c476d981149e1b9488b8f7338bda71df3b01c8 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Mon, 27 Jul 2026 16:59:55 +0530 Subject: [PATCH 3/5] Update manifest.json to include output counters Added output counters to the script execution command and updated source files list. --- statvar_imports/india_ndap/ndap/manifest.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json index 7097abff43..3878626e38 100644 --- a/statvar_imports/india_ndap/ndap/manifest.json +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -9,7 +9,7 @@ "provenance_description": "Life expectancy of a person in India based on gender at state level.", "scripts": [ "download_script.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/India_LifeExpectancy_input.csv --pv_map='India_LifeExpectancy_pvmap.csv' --config_file=India_LifeExpectancy_metadata.csv --output_path=output/India_LifeExpectancy" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/India_LifeExpectancy_input.csv --pv_map='India_LifeExpectancy_pvmap.csv' --config_file=India_LifeExpectancy_metadata.csv --output_path=output/India_LifeExpectancy --output_counters=counters/India_LifeExpectancy_data_counters.csv" ], "import_inputs": [ { @@ -21,7 +21,12 @@ "input_files/India_LifeExpectancy_input.csv" ], "cron_schedule": "00 11 1,15 * *", - "validation_config_file": "validation_config.json" + "validation_config_file": "validation_config.json", + "source_files": [ + "input/*.csv", + "golden_data/*.csv", + "counters/*.csv" + ] } ] -} \ No newline at end of file +} From 30d0c6a85f7ffe8cde31fb8bd2f76ff4816a2c94 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Mon, 27 Jul 2026 18:00:12 +0530 Subject: [PATCH 4/5] adding counters in manifest.json --- statvar_imports/india_ndap/ndap/manifest.json | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json index 3878626e38..315798c403 100644 --- a/statvar_imports/india_ndap/ndap/manifest.json +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -18,15 +18,12 @@ } ], "source_files": [ - "input_files/India_LifeExpectancy_input.csv" - ], - "cron_schedule": "00 11 1,15 * *", - "validation_config_file": "validation_config.json", - "source_files": [ - "input/*.csv", + "input_files/India_LifeExpectancy_input.csv", "golden_data/*.csv", "counters/*.csv" - ] + ], + "cron_schedule": "00 11 1,15 * *", + "validation_config_file": "validation_config.json" } ] } From 8142de251adeede68cd6e9a482544e28a870ce51 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Tue, 28 Jul 2026 16:23:41 +0530 Subject: [PATCH 5/5] Update manifest.json