From 468b780c064d8636d43c91e4eaa87e25b3f475e7 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 10 Aug 2026 09:38:18 +0000 Subject: [PATCH 1/4] Fix download script, add validation config and golden data for NYU Tennessee Diabetes import - Updated download.py to use requests session with browser headers, HTTPAdapter retry policy, and graceful handling of 404 for unreleased years. - Added validation_config.json with deleted records threshold and goldens checks for summary report and observations. - Added golden_data/ with valid Tennessee state and county golden observations and summary report. - Updated manifest.json to include validation_config_file and golden_data source files. --- .../nyu_diabetes/tennessee/download.py | 61 ++++++++++++---- .../golden_data/golden_observations.csv | 71 +++++++++++++++++++ .../golden_data/golden_summary_report.csv | 4 ++ .../nyu_diabetes/tennessee/manifest.json | 9 ++- .../tennessee/validation_config.json | 29 ++++++++ 5 files changed, 157 insertions(+), 17 deletions(-) create mode 100644 statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv create mode 100644 statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv create mode 100644 statvar_imports/nyu_diabetes/tennessee/validation_config.json diff --git a/statvar_imports/nyu_diabetes/tennessee/download.py b/statvar_imports/nyu_diabetes/tennessee/download.py index 6bc33f0b23..ee3515af28 100644 --- a/statvar_imports/nyu_diabetes/tennessee/download.py +++ b/statvar_imports/nyu_diabetes/tennessee/download.py @@ -14,28 +14,51 @@ import os -import requests -from urllib.parse import urlparse -from tqdm import tqdm -from retry import retry -from pathlib import Path from datetime import date -from absl import logging, app -import pandas as pd +from pathlib import Path import re +from urllib.parse import urlparse + +from absl import app, logging +import pandas as pd +import requests +from requests.adapters import HTTPAdapter +from retry import retry +from tqdm import tqdm +from urllib3.util.retry import Retry script_dir = os.path.dirname(os.path.abspath(__file__)) INPUT_DIR = os.path.join(script_dir, "input_files") Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) -@retry(tries=3, delay=5, backoff=2) -def retry_method(url, headers=None): - response = requests.get(url, headers=headers, timeout=120) - response.raise_for_status() - return response +def get_session(): + session = requests.Session() + retries = Retry( + total=5, + backoff_factor=2, + status_forcelist=[429, 500, 502, 503, 504], + raise_on_status=False, + ) + session.mount('https://', HTTPAdapter(max_retries=retries)) + headers = { + 'User-Agent': ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + ), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9' + } + session.headers.update(headers) + return session + +@retry(tries=4, delay=3, backoff=2) +def _fetch_url(session, url): + return session.get(url, timeout=60) def download_files(url_list, save_folder): os.makedirs(os.path.join(save_folder), exist_ok=True) + session = get_session() + downloaded_count = 0 for url in url_list: try: @@ -45,7 +68,12 @@ def download_files(url_list, save_folder): logging.info(f"Downloading: {filename}") - response = retry_method(url) + response = _fetch_url(session, url) + if response.status_code == 404: + logging.info(f"File not yet available (HTTP 404): {url}") + continue + response.raise_for_status() + with response as r: total_size = int(r.headers.get('content-length', 0)) block_size = 1024 @@ -56,8 +84,13 @@ def download_files(url_list, save_folder): f.write(chunk) progress_bar.update(len(chunk)) logging.info(f"Saved: {file_path}\n") + downloaded_count += 1 except Exception as e: - logging.error(f"Failed to download {url} after retries: {e}\n") + logging.error(f"Failed to download {url}: {e}\n") + raise RuntimeError(f"Failed to download required file {url}: {e}") from e + + if downloaded_count == 0: + raise RuntimeError("No files were successfully downloaded.") def generate_urls(start_year, end_year, url_template): url_list = [] diff --git a/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv new file mode 100644 index 0000000000..30fef6a6e2 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv @@ -0,0 +1,71 @@ +"observationAbout" +"geoId/47" +"geoId/47001" +"geoId/47009" +"geoId/47011" +"geoId/47013" +"geoId/47019" +"geoId/47021" +"geoId/47023" +"geoId/47029" +"geoId/47031" +"geoId/47035" +"geoId/47037" +"geoId/47041" +"geoId/47045" +"geoId/47047" +"geoId/47049" +"geoId/47051" +"geoId/47053" +"geoId/47055" +"geoId/47059" +"geoId/47063" +"geoId/47065" +"geoId/47069" +"geoId/47071" +"geoId/47073" +"geoId/47075" +"geoId/47077" +"geoId/47079" +"geoId/47089" +"geoId/47093" +"geoId/47097" +"geoId/47099" +"geoId/47103" +"geoId/47105" +"geoId/47107" +"geoId/47109" +"geoId/47113" +"geoId/47115" +"geoId/47119" +"geoId/47123" +"geoId/47125" +"geoId/47129" +"geoId/47131" +"geoId/47141" +"geoId/47143" +"geoId/47145" +"geoId/47147" +"geoId/47149" +"geoId/47151" +"geoId/47155" +"geoId/47157" +"geoId/47159" +"geoId/47163" +"geoId/47165" +"geoId/47167" +"geoId/47179" +"geoId/47183" +"geoId/47187" +"geoId/47189" +"geoId/47003" +"geoId/47043" +"geoId/47133" +"geoId/47139" +"geoId/47177" +"geoId/47181" +"geoId/47185" +"geoId/47081" +"geoId/47117" +"geoId/47173" +"geoId/47057" diff --git a/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..bd772d75f1 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/golden_data/golden_summary_report.csv @@ -0,0 +1,4 @@ +"StatVar","NumPlaces","Units","MinDate","MeasurementMethods","observationPeriods","ScalingFactors" +"Count_MortalityEvent_DiabetesMellitus","75","[]","2019","[]","[]","[]" +"AgeAdjusted_Count_MortalityEvent_DiabetesMellitus_AsAFractionOf_AgeAdjusted_Count_Person_DiabetesMellitus","75","[Per100000Persons]","2019","[]","[]","[]" +"Count_MortalityEvent_DiabetesMellitus_AsAFractionOf_Count_Person_DiabetesMellitus","75","[Per100000Persons]","2019","[]","[]","[]" diff --git a/statvar_imports/nyu_diabetes/tennessee/manifest.json b/statvar_imports/nyu_diabetes/tennessee/manifest.json index ebb9339cd5..27e0796962 100644 --- a/statvar_imports/nyu_diabetes/tennessee/manifest.json +++ b/statvar_imports/nyu_diabetes/tennessee/manifest.json @@ -6,10 +6,12 @@ "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html", "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", "scripts": ["download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output --output_counters=counters/tennessee_output_counters.csv" ] , "source_files": [ - "input_files/*.xlsx" + "input_files/*.xlsx", + "golden_data/*.csv", + "counters/*.csv" ], "import_inputs": [ { @@ -17,7 +19,8 @@ "cleaned_csv": "output_files/tennessee_output.csv" } ], - "cron_schedule": "0 05 * * 5" + "cron_schedule": "0 05 * * 5", + "validation_config_file": "validation_config.json" } ] } diff --git a/statvar_imports/nyu_diabetes/tennessee/validation_config.json b/statvar_imports/nyu_diabetes/tennessee/validation_config.json new file mode 100644 index 0000000000..f047a1c909 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/validation_config.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "Check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output_files/tennessee_output.csv" + } + } + ] +} From 163d4f518bb174ecb340f7727754067d6c23fb22 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 10 Aug 2026 09:48:56 +0000 Subject: [PATCH 2/4] testing --- .../nyu_diabetes/tennessee/download.py | 61 +++++-------------- 1 file changed, 14 insertions(+), 47 deletions(-) diff --git a/statvar_imports/nyu_diabetes/tennessee/download.py b/statvar_imports/nyu_diabetes/tennessee/download.py index ee3515af28..6bc33f0b23 100644 --- a/statvar_imports/nyu_diabetes/tennessee/download.py +++ b/statvar_imports/nyu_diabetes/tennessee/download.py @@ -14,51 +14,28 @@ import os -from datetime import date -from pathlib import Path -import re -from urllib.parse import urlparse - -from absl import app, logging -import pandas as pd import requests -from requests.adapters import HTTPAdapter +from urllib.parse import urlparse +from tqdm import tqdm from retry import retry -from tqdm import tqdm -from urllib3.util.retry import Retry +from pathlib import Path +from datetime import date +from absl import logging, app +import pandas as pd +import re script_dir = os.path.dirname(os.path.abspath(__file__)) INPUT_DIR = os.path.join(script_dir, "input_files") Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) -def get_session(): - session = requests.Session() - retries = Retry( - total=5, - backoff_factor=2, - status_forcelist=[429, 500, 502, 503, 504], - raise_on_status=False, - ) - session.mount('https://', HTTPAdapter(max_retries=retries)) - headers = { - 'User-Agent': ( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' - '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' - ), - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9' - } - session.headers.update(headers) - return session - -@retry(tries=4, delay=3, backoff=2) -def _fetch_url(session, url): - return session.get(url, timeout=60) +@retry(tries=3, delay=5, backoff=2) +def retry_method(url, headers=None): + response = requests.get(url, headers=headers, timeout=120) + response.raise_for_status() + return response def download_files(url_list, save_folder): os.makedirs(os.path.join(save_folder), exist_ok=True) - session = get_session() - downloaded_count = 0 for url in url_list: try: @@ -68,12 +45,7 @@ def download_files(url_list, save_folder): logging.info(f"Downloading: {filename}") - response = _fetch_url(session, url) - if response.status_code == 404: - logging.info(f"File not yet available (HTTP 404): {url}") - continue - response.raise_for_status() - + response = retry_method(url) with response as r: total_size = int(r.headers.get('content-length', 0)) block_size = 1024 @@ -84,13 +56,8 @@ def download_files(url_list, save_folder): f.write(chunk) progress_bar.update(len(chunk)) logging.info(f"Saved: {file_path}\n") - downloaded_count += 1 except Exception as e: - logging.error(f"Failed to download {url}: {e}\n") - raise RuntimeError(f"Failed to download required file {url}: {e}") from e - - if downloaded_count == 0: - raise RuntimeError("No files were successfully downloaded.") + logging.error(f"Failed to download {url} after retries: {e}\n") def generate_urls(start_year, end_year, url_template): url_list = [] From cead868277fdeb478233e232085a083f209dd200 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 10 Aug 2026 10:32:16 +0000 Subject: [PATCH 3/4] testing --- .../nyu_diabetes/tennessee/download.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/statvar_imports/nyu_diabetes/tennessee/download.py b/statvar_imports/nyu_diabetes/tennessee/download.py index 6bc33f0b23..8f01aa7079 100644 --- a/statvar_imports/nyu_diabetes/tennessee/download.py +++ b/statvar_imports/nyu_diabetes/tennessee/download.py @@ -28,9 +28,21 @@ INPUT_DIR = os.path.join(script_dir, "input_files") Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) -@retry(tries=3, delay=5, backoff=2) +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +class RetryableHTTPError(requests.exceptions.HTTPError): + pass + +@retry(exceptions=(requests.exceptions.ConnectionError, requests.exceptions.Timeout, RetryableHTTPError), tries=3, delay=5, backoff=2) def retry_method(url, headers=None): - response = requests.get(url, headers=headers, timeout=120) + if headers is None: + headers = {} + if "User-Agent" not in headers: + headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + response = requests.get(url, headers=headers, timeout=120, stream=True, verify=False) + if response.status_code >= 500 or response.status_code == 429: + raise RetryableHTTPError(f"Retryable status code {response.status_code}", response=response) response.raise_for_status() return response From 607fd9b5cf944009bf444805dd1c19f17c9038ee Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 10 Aug 2026 11:18:32 +0000 Subject: [PATCH 4/4] testing --- statvar_imports/nyu_diabetes/tennessee/download.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/statvar_imports/nyu_diabetes/tennessee/download.py b/statvar_imports/nyu_diabetes/tennessee/download.py index 8f01aa7079..685e09eedf 100644 --- a/statvar_imports/nyu_diabetes/tennessee/download.py +++ b/statvar_imports/nyu_diabetes/tennessee/download.py @@ -17,7 +17,6 @@ import requests from urllib.parse import urlparse from tqdm import tqdm -from retry import retry from pathlib import Path from datetime import date from absl import logging, app @@ -31,18 +30,12 @@ import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -class RetryableHTTPError(requests.exceptions.HTTPError): - pass - -@retry(exceptions=(requests.exceptions.ConnectionError, requests.exceptions.Timeout, RetryableHTTPError), tries=3, delay=5, backoff=2) def retry_method(url, headers=None): if headers is None: headers = {} if "User-Agent" not in headers: headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" response = requests.get(url, headers=headers, timeout=120, stream=True, verify=False) - if response.status_code >= 500 or response.status_code == 429: - raise RetryableHTTPError(f"Retryable status code {response.status_code}", response=response) response.raise_for_status() return response