diff --git a/statvar_imports/oecd/wastewater_treatment/golden_data/golden_observations.csv b/statvar_imports/oecd/wastewater_treatment/golden_data/golden_observations.csv new file mode 100644 index 0000000000..72312474d8 --- /dev/null +++ b/statvar_imports/oecd/wastewater_treatment/golden_data/golden_observations.csv @@ -0,0 +1,53 @@ +"observationAbout" +"country/AUS" +"country/AUT" +"country/BEL" +"country/CAN" +"country/CHL" +"country/CZE" +"country/DNK" +"country/EST" +"country/FIN" +"country/DEU" +"country/GRC" +"country/HUN" +"country/ISL" +"country/IRL" +"country/ITA" +"country/JPN" +"country/KOR" +"country/LUX" +"country/NLD" +"country/NOR" +"country/POL" +"country/PRT" +"country/SVK" +"country/SVN" +"country/ESP" +"country/SWE" +"country/CHE" +"country/TUR" +"country/GBR" +"country/BGR" +"country/HRV" +"country/ROU" +"country/FRA" +"country/LVA" +"country/USA" +"country/ARG" +"country/MDA" +"country/ZAF" +"country/ARM" +"country/NZL" +"country/ISR" +"country/LTU" +"country/CRI" +"country/BRA" +"country/AZE" +"country/CHN" +"country/GEO" +"country/SAU" +"country/BLR" +"country/IND" +"country/UKR" +"country/MEX" diff --git a/statvar_imports/oecd/wastewater_treatment/golden_data/golden_summary_report.csv b/statvar_imports/oecd/wastewater_treatment/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..4985872130 --- /dev/null +++ b/statvar_imports/oecd/wastewater_treatment/golden_data/golden_summary_report.csv @@ -0,0 +1,9 @@ +"MeasurementMethods","observationPeriods","MinDate","Units","NumPlaces","StatVar","ScalingFactors" +"[]","[]","1970","[Percent]","40","Count_Person_WasteWaterTreatment_TertiaryTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]" +"[]","[]","1970","[Percent]","48","Count_Person_WasteWaterTreatment_WasteWaterNotTreated_PublicSewerage_PublicSewerageWithoutTreatment_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]" +"[]","[]","1970","[Percent]","39","Count_Person_WasteWaterTreatment_UnspecifiedTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]" +"[]","[]","1970","[Percent]","40","Count_Person_WasteWaterTreatment_PrimaryTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]" +"[]","[]","1970","[Percent]","49","Count_Person_WasteWaterTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]" +"[]","[]","1970","[Percent]","36","Count_Person_WasteWaterTreatment_WasteWaterTreated_WasteWaterIndependentlyTreated_AsAFractionOf_Count_Person","[100]" +"[]","[]","1970","[Percent]","42","Count_Person_WasteWaterTreatment_SecondaryTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]" +"[]","[]","1970","[Percent]","51","Count_Person_WasteWaterTreatment_PublicSewerage_AsAFractionOf_Count_Person","[100]" diff --git a/statvar_imports/oecd/wastewater_treatment/manifest.json b/statvar_imports/oecd/wastewater_treatment/manifest.json index cd3821ddbc..de999c7cd5 100644 --- a/statvar_imports/oecd/wastewater_treatment/manifest.json +++ b/statvar_imports/oecd/wastewater_treatment/manifest.json @@ -10,17 +10,20 @@ "scripts": [ "../../../util/download_util_script.py --download_url='https://sdmx.oecd.org/public/rest/data/OECD.ENV.EPI,DSD_WATER_TREAT@DF_WATER_TREAT,/all?dimensionAtObservation=AllDimensions&format=csvfilewithlabels' --output_folder=input", "preprocess.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/oecd_wastewatertreatment_data.csv --pv_map=oecd_wastewatertreatment_pvmap.csv --config_file=oecd_wastewatertreatment_metadata.csv --places_resolved_csv=oecd_wastewatertreatment_places_resolved.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/oecd_wastewatertreatment" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/oecd_wastewatertreatment_data.csv --pv_map=oecd_wastewatertreatment_pvmap.csv --config_file=oecd_wastewatertreatment_metadata.csv --places_resolved_csv=oecd_wastewatertreatment_places_resolved.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/oecd_wastewatertreatment --output_counters=counters/oecd_wastewatertreatment.csv" ], "import_inputs": [ { "template_mcf": "output/oecd_wastewatertreatment.tmcf", - "cleaned_csv": "output/oecd_wastewatertreatment.csv" + "cleaned_csv": "output/oecd_wastewatertreatment_merged.csv" } ], "cron_schedule": "0 10 1,15 * *", + "validation_config_file": "validation_config.json", "source_files": [ - "input/*.csv" + "input/*.csv", + "golden_data/*.csv", + "counters/*.csv" ] } ] diff --git a/statvar_imports/oecd/wastewater_treatment/validation_config.json b/statvar_imports/oecd/wastewater_treatment/validation_config.json new file mode 100644 index 0000000000..735762fb35 --- /dev/null +++ b/statvar_imports/oecd/wastewater_treatment/validation_config.json @@ -0,0 +1,36 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 7 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "check_latest_date_for_all", + "validator": "MAX_DATE_CONSISTENT", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output/oecd_wastewatertreatment_merged.csv" + } + } + ] +}