Fix download script, add validation config and golden data for NYU TennesseDiabetes

statvar_imports/nyu_diabetes/tennessee/download.py

@@ -17,7 +17,6 @@
 import requests
 from urllib.parse import urlparse
 from tqdm import tqdm  
-from retry import retry
 from pathlib import Path
 from datetime import date
 from absl import logging, app
@@ -28,9 +27,15 @@
 INPUT_DIR = os.path.join(script_dir, "input_files")
 Path(INPUT_DIR).mkdir(parents=True, exist_ok=True)
 
-@retry(tries=3, delay=5, backoff=2)
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
 def retry_method(url, headers=None):
-    response = requests.get(url, headers=headers, timeout=120)
+    if headers is None:
+        headers = {}
+    if "User-Agent" not in headers:
+        headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    response = requests.get(url, headers=headers, timeout=120, stream=True, verify=False)
     response.raise_for_status()
     return response
 

statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv

@@ -0,0 +1,71 @@
+"observationAbout"
+"geoId/47"
+"geoId/47001"
+"geoId/47009"
+"geoId/47011"
+"geoId/47013"
+"geoId/47019"
+"geoId/47021"
+"geoId/47023"
+"geoId/47029"
+"geoId/47031"
+"geoId/47035"
+"geoId/47037"
+"geoId/47041"
+"geoId/47045"
+"geoId/47047"
+"geoId/47049"
+"geoId/47051"
+"geoId/47053"
+"geoId/47055"
+"geoId/47059"
+"geoId/47063"
+"geoId/47065"
+"geoId/47069"
+"geoId/47071"
+"geoId/47073"
+"geoId/47075"
+"geoId/47077"
+"geoId/47079"
+"geoId/47089"
+"geoId/47093"
+"geoId/47097"
+"geoId/47099"
+"geoId/47103"
+"geoId/47105"
+"geoId/47107"
+"geoId/47109"
+"geoId/47113"
+"geoId/47115"
+"geoId/47119"
+"geoId/47123"
+"geoId/47125"
+"geoId/47129"
+"geoId/47131"
+"geoId/47141"
+"geoId/47143"
+"geoId/47145"
+"geoId/47147"
+"geoId/47149"
+"geoId/47151"
+"geoId/47155"
+"geoId/47157"
+"geoId/47159"
+"geoId/47163"
+"geoId/47165"
+"geoId/47167"
+"geoId/47179"
+"geoId/47183"
+"geoId/47187"
+"geoId/47189"
+"geoId/47003"
+"geoId/47043"
+"geoId/47133"
+"geoId/47139"
+"geoId/47177"
+"geoId/47181"
+"geoId/47185"
+"geoId/47081"
+"geoId/47117"
+"geoId/47173"
+"geoId/47057"

statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv

@@ -0,0 +1,4 @@
+"StatVar","NumPlaces","Units","MinDate","MeasurementMethods","observationPeriods","ScalingFactors"
+"Count_MortalityEvent_DiabetesMellitus","75","[]","2019","[]","[]","[]"
+"AgeAdjusted_Count_MortalityEvent_DiabetesMellitus_AsAFractionOf_AgeAdjusted_Count_Person_DiabetesMellitus","75","[Per100000Persons]","2019","[]","[]","[]"
+"Count_MortalityEvent_DiabetesMellitus_AsAFractionOf_Count_Person_DiabetesMellitus","75","[Per100000Persons]","2019","[]","[]","[]"

statvar_imports/nyu_diabetes/tennessee/manifest.json

@@ -6,18 +6,21 @@
       "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html",
       "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.",
       "scripts": ["download.py",
-        "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output"
+        "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output --output_counters=counters/tennessee_output_counters.csv"
       ] ,
       "source_files": [
-        "input_files/*.xlsx"
+        "input_files/*.xlsx",
+        "golden_data/*.csv",
+        "counters/*.csv"
       ],
       "import_inputs": [
         {
           "template_mcf": "output_files/tennessee_output.tmcf",
           "cleaned_csv": "output_files/tennessee_output.csv"
         }
       ],
-      "cron_schedule": "0 05 * * 5"
+      "cron_schedule": "0 05 * * 5",
+      "validation_config_file": "validation_config.json"
     }
   ]
 }

statvar_imports/nyu_diabetes/tennessee/validation_config.json

@@ -0,0 +1,29 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0.1
+            }
+        },
+        {
+            "rule_id": "check_goldens_summary_report",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_summary_report.csv"
+            }
+        },
+        {
+            "rule_id": "Check_goldens_output_csv",
+            "description": "Verifies the generated output CSV data matches established critical golden records",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_observations.csv",
+                "input_files": "../../../../output_files/tennessee_output.csv"
+            }
+        }
+    ]
+}