diff --git a/statvar_imports/nyu_diabetes/tennessee/README.md b/statvar_imports/nyu_diabetes/tennessee/README.md index e2324dd2e9..d2bd5f198f 100644 --- a/statvar_imports/nyu_diabetes/tennessee/README.md +++ b/statvar_imports/nyu_diabetes/tennessee/README.md @@ -22,17 +22,35 @@ This import utilizes official mortality data from the Tennessee Department of He ### ⚙️ Workflow -The workflow for this data import involves two main steps: downloading the necessary files and then processing them. +The workflow for this data import involves two main steps: downloading the necessary files and then processing them. The downloader supports TN.gov as the default source and GCS as a semi-manual fallback. #### Step 1: Download the Source Data -To acquire the necessary data files, execute the download script `download.py`. +To download directly from TN.gov, execute: + +```bash +python3 download.py +``` + +When TN.gov blocks the runtime environment, manually upload the source files to: + +```text +gs://unresolved_mcf/nyu_diabetes/tennessee/latest/input_files/ +``` + +Files must use the name `Diabetes_County_YYYY.xlsx`. Download the staged files with: + +```bash +python3 download.py --download_source=gcs +``` + +Both modes check each year from 2019 through the current year. Missing individual GCS files are skipped, and the GCS mode fails if no files are downloaded. The runtime service account must have access to the bucket. All downloaded files will be stored in the directory `input_files`. ### Autorefresh type -This import uses a fully automated refresh process. +The manifest uses the fully automated TN.gov mode. Set its first script to `download.py --download_source=gcs` to enable the semi-manual GCS mode. ----- @@ -53,4 +71,3 @@ The final output is generated by processing the downloaded data using the stat_v --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ --places_resolved_csv=place_resolver.csv ``` - diff --git a/statvar_imports/nyu_diabetes/tennessee/download.py b/statvar_imports/nyu_diabetes/tennessee/download.py index 6bc33f0b23..34ab89145b 100644 --- a/statvar_imports/nyu_diabetes/tennessee/download.py +++ b/statvar_imports/nyu_diabetes/tennessee/download.py @@ -13,16 +13,30 @@ # limitations under the License. +from datetime import date import os -import requests -from urllib.parse import urlparse -from tqdm import tqdm -from retry import retry from pathlib import Path -from datetime import date -from absl import logging, app -import pandas as pd import re +from urllib.parse import urlparse + +from absl import app +from absl import flags +from absl import logging +from google.api_core import exceptions +from google.cloud import storage +import pandas as pd +import requests +from retry import retry +from tqdm import tqdm + +FLAGS = flags.FLAGS + +flags.DEFINE_enum( + 'download_source', + 'tn', + ['tn', 'gcs'], + 'Source from which input files are downloaded.', +) script_dir = os.path.dirname(os.path.abspath(__file__)) INPUT_DIR = os.path.join(script_dir, "input_files") @@ -43,7 +57,8 @@ def download_files(url_list, save_folder): filename = os.path.basename(parsed_url.path) file_path = os.path.join(save_folder, filename) - logging.info(f"Downloading: {filename}") + logging.info( + f"Starting download: source={url}, destination={file_path}") response = retry_method(url) with response as r: @@ -55,14 +70,56 @@ def download_files(url_list, save_folder): for chunk in r.iter_content(block_size): f.write(chunk) progress_bar.update(len(chunk)) - logging.info(f"Saved: {file_path}\n") + file_size = os.path.getsize(file_path) + logging.info( + f"Completed download: source={url}, destination={file_path}, size_bytes={file_size}" + ) except Exception as e: - logging.error(f"Failed to download {url} after retries: {e}\n") + logging.error( + f"Download failed: source={url}, destination={file_path}, error={e}" + ) + + +def download_files_from_gcs(url_list, save_folder): + os.makedirs(save_folder, exist_ok=True) + storage_client = storage.Client() + downloaded_count = 0 + + for url in url_list: + parsed_url = urlparse(url) + filename = os.path.basename(parsed_url.path) + file_path = os.path.join(save_folder, filename) + Path(file_path).unlink(missing_ok=True) + + logging.info( + f"Starting download: source={url}, destination={file_path}") + + try: + blob = storage_client.bucket(parsed_url.netloc).blob( + parsed_url.path.lstrip('/')) + blob.download_to_filename(file_path) + except exceptions.NotFound: + Path(file_path).unlink(missing_ok=True) + logging.warning(f"GCS object not found, skipping: source={url}") + continue + except Exception as e: + Path(file_path).unlink(missing_ok=True) + e.add_note(f"Failed to download GCS object {url}") + raise + + file_size = os.path.getsize(file_path) + logging.info( + f"Completed download: source={url}, destination={file_path}, size_bytes={file_size}" + ) + downloaded_count += 1 + + if downloaded_count == 0: + raise RuntimeError('No files were downloaded from GCS.') def generate_urls(start_year, end_year, url_template): url_list = [] for year in range(start_year,end_year+1): - formatted_url = url_template.format(year,year) + formatted_url = url_template.format(year=year) url_list.append(formatted_url) return url_list @@ -110,12 +167,17 @@ def process_excel_files(input_dir): logging.info("\nExcel file processing complete. 'year' column added to all processed files.") def main(_): - url_template ="https://www.tn.gov/content/dam/tn/health/documents/vital-statistics/death/{}/Diabetes_County_{}.xlsx" + tn_url_template = "https://www.tn.gov/content/dam/tn/health/documents/vital-statistics/death/{year}/Diabetes_County_{year}.xlsx" + gcs_url_template = "gs://unresolved_mcf/nyu_diabetes/tennessee/latest/input_files/Diabetes_County_{year}.xlsx" start_year = 2019 current_year = date.today().year - final_urls = generate_urls(start_year, current_year,url_template) - - download_files(final_urls, save_folder=INPUT_DIR) + + if FLAGS.download_source == 'gcs': + final_urls = generate_urls(start_year, current_year, gcs_url_template) + download_files_from_gcs(final_urls, save_folder=INPUT_DIR) + else: + final_urls = generate_urls(start_year, current_year, tn_url_template) + download_files(final_urls, save_folder=INPUT_DIR) process_excel_files(INPUT_DIR) if __name__ == "__main__": diff --git a/statvar_imports/nyu_diabetes/tennessee/download_test.py b/statvar_imports/nyu_diabetes/tennessee/download_test.py new file mode 100644 index 0000000000..bf588b0258 --- /dev/null +++ b/statvar_imports/nyu_diabetes/tennessee/download_test.py @@ -0,0 +1,98 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from google.api_core import exceptions + +import download + + +class DownloadTest(unittest.TestCase): + + def test_generate_urls_for_tn_and_gcs(self): + tn_template = 'https://tn.example/{year}/Diabetes_County_{year}.xlsx' + gcs_template = 'gs://test-bucket/input/Diabetes_County_{year}.xlsx' + + self.assertEqual( + download.generate_urls(2019, 2020, tn_template), + [ + 'https://tn.example/2019/Diabetes_County_2019.xlsx', + 'https://tn.example/2020/Diabetes_County_2020.xlsx', + ]) + self.assertEqual( + download.generate_urls(2019, 2020, gcs_template), + [ + 'gs://test-bucket/input/Diabetes_County_2019.xlsx', + 'gs://test-bucket/input/Diabetes_County_2020.xlsx', + ]) + + @mock.patch.object(download.storage, 'Client') + def test_gcs_download_skips_missing_file(self, mock_client): + downloaded_blob = mock.Mock() + downloaded_blob.download_to_filename.side_effect = ( + lambda path: Path(path).write_bytes(b'new data')) + missing_blob = mock.Mock() + missing_blob.download_to_filename.side_effect = exceptions.NotFound( + 'not found') + mock_client.return_value.bucket.return_value.blob.side_effect = [ + downloaded_blob, + missing_blob, + ] + urls = [ + 'gs://test-bucket/input/Diabetes_County_2019.xlsx', + 'gs://test-bucket/input/Diabetes_County_2020.xlsx', + ] + + with tempfile.TemporaryDirectory() as temp_dir: + stale_file = Path(temp_dir) / 'Diabetes_County_2020.xlsx' + stale_file.write_bytes(b'stale data') + + download.download_files_from_gcs(urls, temp_dir) + + self.assertEqual( + (Path(temp_dir) / 'Diabetes_County_2019.xlsx').read_bytes(), + b'new data') + self.assertFalse(stale_file.exists()) + + @mock.patch.object(download.storage, 'Client') + def test_gcs_download_fails_when_no_files_exist(self, mock_client): + blob = mock_client.return_value.bucket.return_value.blob.return_value + blob.download_to_filename.side_effect = exceptions.NotFound('not found') + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex(RuntimeError, + 'No files were downloaded from GCS'): + download.download_files_from_gcs( + ['gs://test-bucket/input/Diabetes_County_2019.xlsx'], + temp_dir) + + @mock.patch.object(download.storage, 'Client') + def test_gcs_download_propagates_unexpected_error(self, mock_client): + blob = mock_client.return_value.bucket.return_value.blob.return_value + blob.download_to_filename.side_effect = exceptions.Forbidden( + 'permission denied') + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(exceptions.Forbidden): + download.download_files_from_gcs( + ['gs://test-bucket/input/Diabetes_County_2019.xlsx'], + temp_dir) + + +if __name__ == '__main__': + unittest.main() diff --git a/statvar_imports/nyu_diabetes/tennessee/manifest.json b/statvar_imports/nyu_diabetes/tennessee/manifest.json index ebb9339cd5..6360784e6e 100644 --- a/statvar_imports/nyu_diabetes/tennessee/manifest.json +++ b/statvar_imports/nyu_diabetes/tennessee/manifest.json @@ -21,4 +21,3 @@ } ] } -