diff --git a/statvar_imports/nyu_diabetes/tennessee/download.py b/statvar_imports/nyu_diabetes/tennessee/download.py index 6bc33f0b23..685e09eedf 100644 --- a/statvar_imports/nyu_diabetes/tennessee/download.py +++ b/statvar_imports/nyu_diabetes/tennessee/download.py @@ -17,7 +17,6 @@ import requests from urllib.parse import urlparse from tqdm import tqdm -from retry import retry from pathlib import Path from datetime import date from absl import logging, app @@ -28,9 +27,15 @@ INPUT_DIR = os.path.join(script_dir, "input_files") Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) -@retry(tries=3, delay=5, backoff=2) +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + def retry_method(url, headers=None): - response = requests.get(url, headers=headers, timeout=120) + if headers is None: + headers = {} + if "User-Agent" not in headers: + headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + response = requests.get(url, headers=headers, timeout=120, stream=True, verify=False) response.raise_for_status() return response diff --git a/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv new file mode 100644 index 0000000000..30fef6a6e2 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv @@ -0,0 +1,71 @@ +"observationAbout" +"geoId/47" +"geoId/47001" +"geoId/47009" +"geoId/47011" +"geoId/47013" +"geoId/47019" +"geoId/47021" +"geoId/47023" +"geoId/47029" +"geoId/47031" +"geoId/47035" +"geoId/47037" +"geoId/47041" +"geoId/47045" +"geoId/47047" +"geoId/47049" +"geoId/47051" +"geoId/47053" +"geoId/47055" +"geoId/47059" +"geoId/47063" +"geoId/47065" +"geoId/47069" +"geoId/47071" +"geoId/47073" +"geoId/47075" +"geoId/47077" +"geoId/47079" +"geoId/47089" +"geoId/47093" +"geoId/47097" +"geoId/47099" +"geoId/47103" +"geoId/47105" +"geoId/47107" +"geoId/47109" +"geoId/47113" +"geoId/47115" +"geoId/47119" +"geoId/47123" +"geoId/47125" +"geoId/47129" +"geoId/47131" +"geoId/47141" +"geoId/47143" +"geoId/47145" +"geoId/47147" +"geoId/47149" +"geoId/47151" +"geoId/47155" +"geoId/47157" +"geoId/47159" +"geoId/47163" +"geoId/47165" +"geoId/47167" +"geoId/47179" +"geoId/47183" +"geoId/47187" +"geoId/47189" +"geoId/47003" +"geoId/47043" +"geoId/47133" +"geoId/47139" +"geoId/47177" +"geoId/47181" +"geoId/47185" +"geoId/47081" +"geoId/47117" +"geoId/47173" +"geoId/47057" diff --git a/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..bd772d75f1 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv @@ -0,0 +1,4 @@ +"StatVar","NumPlaces","Units","MinDate","MeasurementMethods","observationPeriods","ScalingFactors" +"Count_MortalityEvent_DiabetesMellitus","75","[]","2019","[]","[]","[]" +"AgeAdjusted_Count_MortalityEvent_DiabetesMellitus_AsAFractionOf_AgeAdjusted_Count_Person_DiabetesMellitus","75","[Per100000Persons]","2019","[]","[]","[]" +"Count_MortalityEvent_DiabetesMellitus_AsAFractionOf_Count_Person_DiabetesMellitus","75","[Per100000Persons]","2019","[]","[]","[]" diff --git a/statvar_imports/nyu_diabetes/tennessee/manifest.json b/statvar_imports/nyu_diabetes/tennessee/manifest.json index ebb9339cd5..27e0796962 100644 --- a/statvar_imports/nyu_diabetes/tennessee/manifest.json +++ b/statvar_imports/nyu_diabetes/tennessee/manifest.json @@ -6,10 +6,12 @@ "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html", "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", "scripts": ["download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output --output_counters=counters/tennessee_output_counters.csv" ] , "source_files": [ - "input_files/*.xlsx" + "input_files/*.xlsx", + "golden_data/*.csv", + "counters/*.csv" ], "import_inputs": [ { @@ -17,7 +19,8 @@ "cleaned_csv": "output_files/tennessee_output.csv" } ], - "cron_schedule": "0 05 * * 5" + "cron_schedule": "0 05 * * 5", + "validation_config_file": "validation_config.json" } ] } diff --git a/statvar_imports/nyu_diabetes/tennessee/validation_config.json b/statvar_imports/nyu_diabetes/tennessee/validation_config.json new file mode 100644 index 0000000000..f047a1c909 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/validation_config.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "Check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output_files/tennessee_output.csv" + } + } + ] +}