Skip to content

Add retries with backoff and increase timeout in statistics_poland download script #2157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
abhishekjaisw wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Add retries with backoff and increase timeout in statistics_poland download script #2157

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5e6b813
Add retries with backoff and increase timeout for Statistics Poland d…
abhishekjaisw Aug 13, 2026
87eed1e
Update make_request to retry on 429 rate limit and 5xx errors with ba…
abhishekjaisw Aug 13, 2026
f905d4c
Address review comments: simplify make_request and configure 10 retri…
abhishekjaisw Aug 13, 2026
84f1a86
Merge branch 'master' into statistics_poland_retry_timeout_fix
abhishekjaisw Aug 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 46 additions & 10 deletions statvar_imports/statistics_poland/download_input_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,28 +70,63 @@ def get_template_map(template_df):
name_to_code[clean_name] = str(code).strip()
return name_to_code

def fetch_variables():
"""Fetches all variables for Subject P3447."""
from urllib3.util import Retry
from requests.adapters import HTTPAdapter

def get_http_session(retries=5, backoff_factor=2):
"""Creates a requests session with automatic HTTP retries and backoff."""
session = requests.Session()
retry_strategy = Retry(
total=retries,
backoff_factor=backoff_factor,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["GET"],
raise_on_status=False
)
adapter = HTTPAdapter(max_retries=retry_strategy)
session.mount("https://", adapter)
session.mount("http://", adapter)
return session

def make_request(session, url, headers=None, params=None, timeout=60, max_attempts=5):
"""Makes an HTTP GET request with exponential backoff on timeouts/connection errors."""
for attempt in range(1, max_attempts + 1):
try:
resp = session.get(url, headers=headers, params=params, timeout=timeout)
return resp
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.RequestException) as e:
logging.warning(f"Request attempt {attempt}/{max_attempts} failed for {url}: {e}")
if attempt == max_attempts:
raise
time.sleep(2 ** attempt)
return None

@gemini-code-assist gemini-code-assist Bot Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The current implementation of make_request introduces a redundant double-retry loop. Since the requests.Session is already configured with an HTTPAdapter and a Retry strategy, wrapping session.get in a manual retry loop will cause nested retries. We should simplify make_request to rely entirely on the session's built-in retry strategy. Additionally, when downloading data from unstable or unreliable external sources like Statistics Poland, ensure that the session's retry strategy is configured with a higher number of retries (e.g., 10 retries) to guarantee download success, even if it results in a long cumulative wait time.

def make_request(session, url, headers=None, params=None, timeout=60):
    # Ensure the session's retry strategy is configured with a high number of retries (e.g., 10)
    return session.get(url, headers=headers, params=params, timeout=timeout)
References
  1. When downloading data from unstable or unreliable external sources, configure a higher number of retries (e.g., tries=10) to ensure download success, even if it results in a long cumulative wait time.

@abhishekjaisw abhishekjaisw Aug 13, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in commit f905d4c. Simplified make_request to rely directly on the session's built-in HTTPAdapter with urllib3.util.Retry and configured 10 retries with exponential backoff on all status codes and timeouts.


def fetch_variables(session):
"""Fetches all variables for Subject P3447 with retries."""
logging.info(f"Downloading variable list for Subject {SUBJECT_ID}...")
v_map = {}

for page in range(10):
url = f"{API_BASE_URL}/variables?subject-id={SUBJECT_ID}&page-size=100&lang=pl&page={page}"
try:
resp = requests.get(url, headers=HEADERS, timeout=20)
if resp.status_code != 200: break
resp = make_request(session, url, headers=HEADERS, timeout=60)
if resp is None or resp.status_code != 200:
logging.warning(f"Metadata page {page} returned status {resp.status_code if resp else 'None'}")
break
data = resp.json()
results = data.get('results', [])
if not results: break
if not results:
break

for item in results:
full_name_parts = [str(v) for k, v in item.items() if k.startswith('n') and v]
full_name = " ".join(full_name_parts).lower()
v_map[str(item['id'])] = full_name

if len(results) < 100: break
if len(results) < 100:
break
except Exception as e:
logging.error(f"Metadata error page {page}: {e}")
logging.error(f"Metadata error page {page} after retries: {e}")
break

logging.info(f"Indexed {len(v_map)} variables.")
Expand All @@ -110,8 +145,9 @@ def download_and_process():
template_df.index.levels[1].astype(str)
])

session = get_http_session()
region_map = get_template_map(template_df)
v_metadata = fetch_variables()
v_metadata = fetch_variables(session)
if not v_metadata:
raise ValueError("Variable metadata failed to download.")

Expand Down Expand Up @@ -160,8 +196,8 @@ def download_and_process():
params.append(('year', str(y)))

try:
resp = requests.get(api_url, headers=HEADERS, params=params, timeout=20)
if resp.status_code != 200: continue
resp = make_request(session, api_url, headers=HEADERS, params=params, timeout=60)
if resp is None or resp.status_code != 200: continue
results = resp.json().get('results', [])
if not results: continue

Expand Down
Loading