OECD_Waste_Water_Treatment cloud job fix and goldens

statvar_imports/oecd/wastewater_treatment/golden_data/golden_observations.csv

@@ -0,0 +1,53 @@
+"observationAbout"
+"country/AUS"
+"country/AUT"
+"country/BEL"
+"country/CAN"
+"country/CHL"
+"country/CZE"
+"country/DNK"
+"country/EST"
+"country/FIN"
+"country/DEU"
+"country/GRC"
+"country/HUN"
+"country/ISL"
+"country/IRL"
+"country/ITA"
+"country/JPN"
+"country/KOR"
+"country/LUX"
+"country/NLD"
+"country/NOR"
+"country/POL"
+"country/PRT"
+"country/SVK"
+"country/SVN"
+"country/ESP"
+"country/SWE"
+"country/CHE"
+"country/TUR"
+"country/GBR"
+"country/BGR"
+"country/HRV"
+"country/ROU"
+"country/FRA"
+"country/LVA"
+"country/USA"
+"country/ARG"
+"country/MDA"
+"country/ZAF"
+"country/ARM"
+"country/NZL"
+"country/ISR"
+"country/LTU"
+"country/CRI"
+"country/BRA"
+"country/AZE"
+"country/CHN"
+"country/GEO"
+"country/SAU"
+"country/BLR"
+"country/IND"
+"country/UKR"
+"country/MEX"

statvar_imports/oecd/wastewater_treatment/golden_data/golden_summary_report.csv

@@ -0,0 +1,9 @@
+"MeasurementMethods","observationPeriods","MinDate","Units","NumPlaces","StatVar","ScalingFactors"
+"[]","[]","1970","[Percent]","40","Count_Person_WasteWaterTreatment_TertiaryTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]"
+"[]","[]","1970","[Percent]","48","Count_Person_WasteWaterTreatment_WasteWaterNotTreated_PublicSewerage_PublicSewerageWithoutTreatment_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]"
+"[]","[]","1970","[Percent]","39","Count_Person_WasteWaterTreatment_UnspecifiedTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]"
+"[]","[]","1970","[Percent]","40","Count_Person_WasteWaterTreatment_PrimaryTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]"
+"[]","[]","1970","[Percent]","49","Count_Person_WasteWaterTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]"
+"[]","[]","1970","[Percent]","36","Count_Person_WasteWaterTreatment_WasteWaterTreated_WasteWaterIndependentlyTreated_AsAFractionOf_Count_Person","[100]"
+"[]","[]","1970","[Percent]","42","Count_Person_WasteWaterTreatment_SecondaryTreatment_WasteWaterTreated_PublicSewerage_WasteWaterTreatmentPlant_AsAFractionOf_Count_Person_WasteWaterTreatment_PublicSewerage","[100]"
+"[]","[]","1970","[Percent]","51","Count_Person_WasteWaterTreatment_PublicSewerage_AsAFractionOf_Count_Person","[100]"

statvar_imports/oecd/wastewater_treatment/manifest.json

@@ -10,17 +10,20 @@
             "scripts": [
                 "../../../util/download_util_script.py --download_url='https://sdmx.oecd.org/public/rest/data/OECD.ENV.EPI,DSD_WATER_TREAT@DF_WATER_TREAT,/all?dimensionAtObservation=AllDimensions&format=csvfilewithlabels' --output_folder=input",
                 "preprocess.py",
-                "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/oecd_wastewatertreatment_data.csv --pv_map=oecd_wastewatertreatment_pvmap.csv --config_file=oecd_wastewatertreatment_metadata.csv --places_resolved_csv=oecd_wastewatertreatment_places_resolved.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/oecd_wastewatertreatment"
+                "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/oecd_wastewatertreatment_data.csv --pv_map=oecd_wastewatertreatment_pvmap.csv --config_file=oecd_wastewatertreatment_metadata.csv --places_resolved_csv=oecd_wastewatertreatment_places_resolved.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/oecd_wastewatertreatment --output_counters=counters/oecd_wastewatertreatment.csv"
             ],
             "import_inputs": [
                 {
                     "template_mcf": "output/oecd_wastewatertreatment.tmcf",
-                    "cleaned_csv": "output/oecd_wastewatertreatment.csv"
+                    "cleaned_csv": "output/oecd_wastewatertreatment_merged.csv"
                 }
             ],
             "cron_schedule": "0 10 1,15 * *",
+            "validation_config_file": "validation_config.json",
             "source_files": [
-                "input/*.csv"
+                "input/*.csv",
+                "golden_data/*.csv",
+                "counters/*.csv"
             ]
         }
     ]

statvar_imports/oecd/wastewater_treatment/validation_config.json

@@ -0,0 +1,36 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 7
+            }
+        },
+        {
+            "rule_id": "check_goldens_summary_report",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_summary_report.csv"
+            }
+        },
+        {
+            "rule_id": "check_latest_date_for_all",
+            "validator": "MAX_DATE_CONSISTENT",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_summary_report.csv"
+            }
+        },
+        {
+            "rule_id": "check_goldens_output_csv",
+            "description": "Verifies the generated output CSV data matches established critical golden records",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_observations.csv",
+                "input_files": "../../../../output/oecd_wastewatertreatment_merged.csv"
+            }
+        }
+    ]
+}