From 4c84b34883a82a1b6f850183a5ee38d92028e9da Mon Sep 17 00:00:00 2001 From: rohit kumar Date: Fri, 14 Aug 2026 06:33:47 +0000 Subject: [PATCH 1/2] Add GCS download mode for Tennessee diabetes --- .../nyu_diabetes/tennessee/README.md | 25 ++++- .../nyu_diabetes/tennessee/download.py | 92 ++++++++++++++--- .../nyu_diabetes/tennessee/download_test.py | 98 +++++++++++++++++++ .../nyu_diabetes/tennessee/manifest.json | 3 +- 4 files changed, 197 insertions(+), 21 deletions(-) create mode 100644 statvar_imports/nyu_diabetes/tennessee/download_test.py diff --git a/statvar_imports/nyu_diabetes/tennessee/README.md b/statvar_imports/nyu_diabetes/tennessee/README.md index e2324dd2e9..7b6667698c 100644 --- a/statvar_imports/nyu_diabetes/tennessee/README.md +++ b/statvar_imports/nyu_diabetes/tennessee/README.md @@ -22,17 +22,35 @@ This import utilizes official mortality data from the Tennessee Department of He ### ⚙️ Workflow -The workflow for this data import involves two main steps: downloading the necessary files and then processing them. +The workflow for this data import involves two main steps: downloading the necessary files and then processing them. The downloader supports TN.gov as the default source and GCS as a semi-manual fallback. #### Step 1: Download the Source Data -To acquire the necessary data files, execute the download script `download.py`. +To download directly from TN.gov, execute: + +```bash +python3 download.py +``` + +When TN.gov blocks the runtime environment, manually upload the source files to: + +```text +gs://unresolved_mcf/nyu_diabetes/tennessee/latest/input_files/ +``` + +Files must use the name `Diabetes_County_YYYY.xlsx`. Download the staged files with: + +```bash +python3 download.py --download_source=gcs +``` + +Both modes check each year from 2019 through the current year. Missing individual GCS files are skipped, and the GCS mode fails if no files are downloaded. The runtime service account must have access to the bucket. All downloaded files will be stored in the directory `input_files`. ### Autorefresh type -This import uses a fully automated refresh process. +The manifest uses the semi-manual GCS mode. Run `python3 download.py` directly to use TN.gov instead. ----- @@ -53,4 +71,3 @@ The final output is generated by processing the downloaded data using the stat_v --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ --places_resolved_csv=place_resolver.csv ``` - diff --git a/statvar_imports/nyu_diabetes/tennessee/download.py b/statvar_imports/nyu_diabetes/tennessee/download.py index 6bc33f0b23..34ab89145b 100644 --- a/statvar_imports/nyu_diabetes/tennessee/download.py +++ b/statvar_imports/nyu_diabetes/tennessee/download.py @@ -13,16 +13,30 @@ # limitations under the License. +from datetime import date import os -import requests -from urllib.parse import urlparse -from tqdm import tqdm -from retry import retry from pathlib import Path -from datetime import date -from absl import logging, app -import pandas as pd import re +from urllib.parse import urlparse + +from absl import app +from absl import flags +from absl import logging +from google.api_core import exceptions +from google.cloud import storage +import pandas as pd +import requests +from retry import retry +from tqdm import tqdm + +FLAGS = flags.FLAGS + +flags.DEFINE_enum( + 'download_source', + 'tn', + ['tn', 'gcs'], + 'Source from which input files are downloaded.', +) script_dir = os.path.dirname(os.path.abspath(__file__)) INPUT_DIR = os.path.join(script_dir, "input_files") @@ -43,7 +57,8 @@ def download_files(url_list, save_folder): filename = os.path.basename(parsed_url.path) file_path = os.path.join(save_folder, filename) - logging.info(f"Downloading: {filename}") + logging.info( + f"Starting download: source={url}, destination={file_path}") response = retry_method(url) with response as r: @@ -55,14 +70,56 @@ def download_files(url_list, save_folder): for chunk in r.iter_content(block_size): f.write(chunk) progress_bar.update(len(chunk)) - logging.info(f"Saved: {file_path}\n") + file_size = os.path.getsize(file_path) + logging.info( + f"Completed download: source={url}, destination={file_path}, size_bytes={file_size}" + ) except Exception as e: - logging.error(f"Failed to download {url} after retries: {e}\n") + logging.error( + f"Download failed: source={url}, destination={file_path}, error={e}" + ) + + +def download_files_from_gcs(url_list, save_folder): + os.makedirs(save_folder, exist_ok=True) + storage_client = storage.Client() + downloaded_count = 0 + + for url in url_list: + parsed_url = urlparse(url) + filename = os.path.basename(parsed_url.path) + file_path = os.path.join(save_folder, filename) + Path(file_path).unlink(missing_ok=True) + + logging.info( + f"Starting download: source={url}, destination={file_path}") + + try: + blob = storage_client.bucket(parsed_url.netloc).blob( + parsed_url.path.lstrip('/')) + blob.download_to_filename(file_path) + except exceptions.NotFound: + Path(file_path).unlink(missing_ok=True) + logging.warning(f"GCS object not found, skipping: source={url}") + continue + except Exception as e: + Path(file_path).unlink(missing_ok=True) + e.add_note(f"Failed to download GCS object {url}") + raise + + file_size = os.path.getsize(file_path) + logging.info( + f"Completed download: source={url}, destination={file_path}, size_bytes={file_size}" + ) + downloaded_count += 1 + + if downloaded_count == 0: + raise RuntimeError('No files were downloaded from GCS.') def generate_urls(start_year, end_year, url_template): url_list = [] for year in range(start_year,end_year+1): - formatted_url = url_template.format(year,year) + formatted_url = url_template.format(year=year) url_list.append(formatted_url) return url_list @@ -110,12 +167,17 @@ def process_excel_files(input_dir): logging.info("\nExcel file processing complete. 'year' column added to all processed files.") def main(_): - url_template ="https://www.tn.gov/content/dam/tn/health/documents/vital-statistics/death/{}/Diabetes_County_{}.xlsx" + tn_url_template = "https://www.tn.gov/content/dam/tn/health/documents/vital-statistics/death/{year}/Diabetes_County_{year}.xlsx" + gcs_url_template = "gs://unresolved_mcf/nyu_diabetes/tennessee/latest/input_files/Diabetes_County_{year}.xlsx" start_year = 2019 current_year = date.today().year - final_urls = generate_urls(start_year, current_year,url_template) - - download_files(final_urls, save_folder=INPUT_DIR) + + if FLAGS.download_source == 'gcs': + final_urls = generate_urls(start_year, current_year, gcs_url_template) + download_files_from_gcs(final_urls, save_folder=INPUT_DIR) + else: + final_urls = generate_urls(start_year, current_year, tn_url_template) + download_files(final_urls, save_folder=INPUT_DIR) process_excel_files(INPUT_DIR) if __name__ == "__main__": diff --git a/statvar_imports/nyu_diabetes/tennessee/download_test.py b/statvar_imports/nyu_diabetes/tennessee/download_test.py new file mode 100644 index 0000000000..bf588b0258 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/download_test.py @@ -0,0 +1,98 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from google.api_core import exceptions + +import download + + +class DownloadTest(unittest.TestCase): + + def test_generate_urls_for_tn_and_gcs(self): + tn_template = 'https://tn.example/{year}/Diabetes_County_{year}.xlsx' + gcs_template = 'gs://test-bucket/input/Diabetes_County_{year}.xlsx' + + self.assertEqual( + download.generate_urls(2019, 2020, tn_template), + [ + 'https://tn.example/2019/Diabetes_County_2019.xlsx', + 'https://tn.example/2020/Diabetes_County_2020.xlsx', + ]) + self.assertEqual( + download.generate_urls(2019, 2020, gcs_template), + [ + 'gs://test-bucket/input/Diabetes_County_2019.xlsx', + 'gs://test-bucket/input/Diabetes_County_2020.xlsx', + ]) + + @mock.patch.object(download.storage, 'Client') + def test_gcs_download_skips_missing_file(self, mock_client): + downloaded_blob = mock.Mock() + downloaded_blob.download_to_filename.side_effect = ( + lambda path: Path(path).write_bytes(b'new data')) + missing_blob = mock.Mock() + missing_blob.download_to_filename.side_effect = exceptions.NotFound( + 'not found') + mock_client.return_value.bucket.return_value.blob.side_effect = [ + downloaded_blob, + missing_blob, + ] + urls = [ + 'gs://test-bucket/input/Diabetes_County_2019.xlsx', + 'gs://test-bucket/input/Diabetes_County_2020.xlsx', + ] + + with tempfile.TemporaryDirectory() as temp_dir: + stale_file = Path(temp_dir) / 'Diabetes_County_2020.xlsx' + stale_file.write_bytes(b'stale data') + + download.download_files_from_gcs(urls, temp_dir) + + self.assertEqual( + (Path(temp_dir) / 'Diabetes_County_2019.xlsx').read_bytes(), + b'new data') + self.assertFalse(stale_file.exists()) + + @mock.patch.object(download.storage, 'Client') + def test_gcs_download_fails_when_no_files_exist(self, mock_client): + blob = mock_client.return_value.bucket.return_value.blob.return_value + blob.download_to_filename.side_effect = exceptions.NotFound('not found') + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex(RuntimeError, + 'No files were downloaded from GCS'): + download.download_files_from_gcs( + ['gs://test-bucket/input/Diabetes_County_2019.xlsx'], + temp_dir) + + @mock.patch.object(download.storage, 'Client') + def test_gcs_download_propagates_unexpected_error(self, mock_client): + blob = mock_client.return_value.bucket.return_value.blob.return_value + blob.download_to_filename.side_effect = exceptions.Forbidden( + 'permission denied') + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(exceptions.Forbidden): + download.download_files_from_gcs( + ['gs://test-bucket/input/Diabetes_County_2019.xlsx'], + temp_dir) + + +if __name__ == '__main__': + unittest.main() diff --git a/statvar_imports/nyu_diabetes/tennessee/manifest.json b/statvar_imports/nyu_diabetes/tennessee/manifest.json index ebb9339cd5..a560b4045f 100644 --- a/statvar_imports/nyu_diabetes/tennessee/manifest.json +++ b/statvar_imports/nyu_diabetes/tennessee/manifest.json @@ -5,7 +5,7 @@ "curator_emails": ["support@datacommons.org"], "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html", "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", - "scripts": ["download.py", + "scripts": ["download.py --download_source=gcs", "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" ] , "source_files": [ @@ -21,4 +21,3 @@ } ] } - From bf82466326cd2957822ec8815ee397ff451f6cf0 Mon Sep 17 00:00:00 2001 From: rohit kumar Date: Fri, 14 Aug 2026 14:51:28 +0000 Subject: [PATCH 2/2] reverted default to download from api --- statvar_imports/nyu_diabetes/tennessee/README.md | 2 +- statvar_imports/nyu_diabetes/tennessee/manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/statvar_imports/nyu_diabetes/tennessee/README.md b/statvar_imports/nyu_diabetes/tennessee/README.md index 7b6667698c..d2bd5f198f 100644 --- a/statvar_imports/nyu_diabetes/tennessee/README.md +++ b/statvar_imports/nyu_diabetes/tennessee/README.md @@ -50,7 +50,7 @@ All downloaded files will be stored in the directory `input_files`. ### Autorefresh type -The manifest uses the semi-manual GCS mode. Run `python3 download.py` directly to use TN.gov instead. +The manifest uses the fully automated TN.gov mode. Set its first script to `download.py --download_source=gcs` to enable the semi-manual GCS mode. ----- diff --git a/statvar_imports/nyu_diabetes/tennessee/manifest.json b/statvar_imports/nyu_diabetes/tennessee/manifest.json index a560b4045f..6360784e6e 100644 --- a/statvar_imports/nyu_diabetes/tennessee/manifest.json +++ b/statvar_imports/nyu_diabetes/tennessee/manifest.json @@ -5,7 +5,7 @@ "curator_emails": ["support@datacommons.org"], "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html", "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", - "scripts": ["download.py --download_source=gcs", + "scripts": ["download.py", "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" ] , "source_files": [