From 5e6b813c796407833ba8d3ee95010baede593152 Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Thu, 13 Aug 2026 11:37:42 +0000 Subject: [PATCH 1/3] Add retries with backoff and increase timeout for Statistics Poland download script --- .../statistics_poland/download_input_data.py | 56 +++++++++++++++---- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/statvar_imports/statistics_poland/download_input_data.py b/statvar_imports/statistics_poland/download_input_data.py index 4069186c33..50f133aeae 100644 --- a/statvar_imports/statistics_poland/download_input_data.py +++ b/statvar_imports/statistics_poland/download_input_data.py @@ -70,28 +70,63 @@ def get_template_map(template_df): name_to_code[clean_name] = str(code).strip() return name_to_code -def fetch_variables(): - """Fetches all variables for Subject P3447.""" +from urllib3.util import Retry +from requests.adapters import HTTPAdapter + +def get_http_session(retries=5, backoff_factor=2): + """Creates a requests session with automatic HTTP retries and backoff.""" + session = requests.Session() + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["GET"], + raise_on_status=False + ) + adapter = HTTPAdapter(max_retries=retry_strategy) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + +def make_request(session, url, headers=None, params=None, timeout=60, max_attempts=5): + """Makes an HTTP GET request with exponential backoff on timeouts/connection errors.""" + for attempt in range(1, max_attempts + 1): + try: + resp = session.get(url, headers=headers, params=params, timeout=timeout) + return resp + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.RequestException) as e: + logging.warning(f"Request attempt {attempt}/{max_attempts} failed for {url}: {e}") + if attempt == max_attempts: + raise + time.sleep(2 ** attempt) + return None + +def fetch_variables(session): + """Fetches all variables for Subject P3447 with retries.""" logging.info(f"Downloading variable list for Subject {SUBJECT_ID}...") v_map = {} for page in range(10): url = f"{API_BASE_URL}/variables?subject-id={SUBJECT_ID}&page-size=100&lang=pl&page={page}" try: - resp = requests.get(url, headers=HEADERS, timeout=20) - if resp.status_code != 200: break + resp = make_request(session, url, headers=HEADERS, timeout=60) + if resp is None or resp.status_code != 200: + logging.warning(f"Metadata page {page} returned status {resp.status_code if resp else 'None'}") + break data = resp.json() results = data.get('results', []) - if not results: break + if not results: + break for item in results: full_name_parts = [str(v) for k, v in item.items() if k.startswith('n') and v] full_name = " ".join(full_name_parts).lower() v_map[str(item['id'])] = full_name - if len(results) < 100: break + if len(results) < 100: + break except Exception as e: - logging.error(f"Metadata error page {page}: {e}") + logging.error(f"Metadata error page {page} after retries: {e}") break logging.info(f"Indexed {len(v_map)} variables.") @@ -110,8 +145,9 @@ def download_and_process(): template_df.index.levels[1].astype(str) ]) + session = get_http_session() region_map = get_template_map(template_df) - v_metadata = fetch_variables() + v_metadata = fetch_variables(session) if not v_metadata: raise ValueError("Variable metadata failed to download.") @@ -160,8 +196,8 @@ def download_and_process(): params.append(('year', str(y))) try: - resp = requests.get(api_url, headers=HEADERS, params=params, timeout=20) - if resp.status_code != 200: continue + resp = make_request(session, api_url, headers=HEADERS, params=params, timeout=60) + if resp is None or resp.status_code != 200: continue results = resp.json().get('results', []) if not results: continue From 87eed1e65fc05523278e210190e7c0135672bb2f Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Thu, 13 Aug 2026 13:03:29 +0000 Subject: [PATCH 2/3] Update make_request to retry on 429 rate limit and 5xx errors with backoff --- .../statistics_poland/download_input_data.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/statvar_imports/statistics_poland/download_input_data.py b/statvar_imports/statistics_poland/download_input_data.py index 50f133aeae..92081eef8b 100644 --- a/statvar_imports/statistics_poland/download_input_data.py +++ b/statvar_imports/statistics_poland/download_input_data.py @@ -89,10 +89,18 @@ def get_http_session(retries=5, backoff_factor=2): return session def make_request(session, url, headers=None, params=None, timeout=60, max_attempts=5): - """Makes an HTTP GET request with exponential backoff on timeouts/connection errors.""" + """Makes an HTTP GET request with exponential backoff on timeouts, connection errors, and 429/5xx status codes.""" for attempt in range(1, max_attempts + 1): try: resp = session.get(url, headers=headers, params=params, timeout=timeout) + if resp.status_code == 200: + return resp + if resp.status_code in [429, 500, 502, 503, 504]: + retry_after = int(resp.headers.get('Retry-After', 2 ** attempt)) + logging.warning(f"HTTP {resp.status_code} received for {url} (attempt {attempt}/{max_attempts}). Backing off for {retry_after}s...") + time.sleep(retry_after) + continue + logging.warning(f"HTTP {resp.status_code} received for {url}: {resp.text[:200]}") return resp except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.RequestException) as e: logging.warning(f"Request attempt {attempt}/{max_attempts} failed for {url}: {e}") @@ -197,7 +205,9 @@ def download_and_process(): try: resp = make_request(session, api_url, headers=HEADERS, params=params, timeout=60) - if resp is None or resp.status_code != 200: continue + if resp is None or resp.status_code != 200: + logging.error(f"Download returned status {resp.status_code if resp else 'None'} for var {var_id} level {lv}") + continue results = resp.json().get('results', []) if not results: continue @@ -229,7 +239,8 @@ def download_and_process(): }) except Exception as e: logging.error(f"Download Error on {var_id}: {e}") - time.sleep(0.05) + time.sleep(0.1) + time.sleep(0.1) if not master_data: raise ValueError("No data collected during the download loop.") From f905d4c9b6cea590ae90e4958653ce24356d8af4 Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Thu, 13 Aug 2026 13:26:38 +0000 Subject: [PATCH 3/3] Address review comments: simplify make_request and configure 10 retries in session --- .../statistics_poland/download_input_data.py | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/statvar_imports/statistics_poland/download_input_data.py b/statvar_imports/statistics_poland/download_input_data.py index 92081eef8b..5aab156099 100644 --- a/statvar_imports/statistics_poland/download_input_data.py +++ b/statvar_imports/statistics_poland/download_input_data.py @@ -8,6 +8,8 @@ from datetime import datetime from google.cloud import storage import io +from urllib3.util import Retry +from requests.adapters import HTTPAdapter # Configure logging logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') @@ -70,14 +72,14 @@ def get_template_map(template_df): name_to_code[clean_name] = str(code).strip() return name_to_code -from urllib3.util import Retry -from requests.adapters import HTTPAdapter - -def get_http_session(retries=5, backoff_factor=2): +def get_http_session(retries=10, backoff_factor=1.5): """Creates a requests session with automatic HTTP retries and backoff.""" session = requests.Session() retry_strategy = Retry( total=retries, + connect=retries, + read=retries, + status=retries, backoff_factor=backoff_factor, status_forcelist=[429, 500, 502, 503, 504], allowed_methods=["GET"], @@ -88,26 +90,13 @@ def get_http_session(retries=5, backoff_factor=2): session.mount("http://", adapter) return session -def make_request(session, url, headers=None, params=None, timeout=60, max_attempts=5): - """Makes an HTTP GET request with exponential backoff on timeouts, connection errors, and 429/5xx status codes.""" - for attempt in range(1, max_attempts + 1): - try: - resp = session.get(url, headers=headers, params=params, timeout=timeout) - if resp.status_code == 200: - return resp - if resp.status_code in [429, 500, 502, 503, 504]: - retry_after = int(resp.headers.get('Retry-After', 2 ** attempt)) - logging.warning(f"HTTP {resp.status_code} received for {url} (attempt {attempt}/{max_attempts}). Backing off for {retry_after}s...") - time.sleep(retry_after) - continue - logging.warning(f"HTTP {resp.status_code} received for {url}: {resp.text[:200]}") - return resp - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.RequestException) as e: - logging.warning(f"Request attempt {attempt}/{max_attempts} failed for {url}: {e}") - if attempt == max_attempts: - raise - time.sleep(2 ** attempt) - return None +def make_request(session, url, headers=None, params=None, timeout=60): + """Makes an HTTP GET request using the session's configured retry strategy and timeout.""" + try: + return session.get(url, headers=headers, params=params, timeout=timeout) + except Exception as e: + logging.error(f"HTTP GET failed for {url}: {e}") + return None def fetch_variables(session): """Fetches all variables for Subject P3447 with retries."""