Add GCS download mode for Tennessee diabetes import

statvar_imports/nyu_diabetes/tennessee/README.md

@@ -22,17 +22,35 @@ This import utilizes official mortality data from the Tennessee Department of He
 
 ### ⚙️ Workflow
 
-The workflow for this data import involves two main steps: downloading the necessary files and then processing them.
+The workflow for this data import involves two main steps: downloading the necessary files and then processing them. The downloader supports TN.gov as the default source and GCS as a semi-manual fallback.
 
 #### Step 1: Download the Source Data
 
-To acquire the necessary data files, execute the download script `download.py`.
+To download directly from TN.gov, execute:
+
+```bash
+python3 download.py
+```
+
+When TN.gov blocks the runtime environment, manually upload the source files to:
+
+```text
+gs://unresolved_mcf/nyu_diabetes/tennessee/latest/input_files/
+```
+
+Files must use the name `Diabetes_County_YYYY.xlsx`. Download the staged files with:
+
+```bash
+python3 download.py --download_source=gcs
+```
+
+Both modes check each year from 2019 through the current year. Missing individual GCS files are skipped, and the GCS mode fails if no files are downloaded. The runtime service account must have access to the bucket.
 
 All downloaded files will be stored in the directory `input_files`.
 
 ### Autorefresh type
 
-This import uses a fully automated refresh process.
+The manifest uses the semi-manual GCS mode. Run `python3 download.py` directly to use TN.gov instead.
 
 -----
 
@@ -53,4 +71,3 @@ The final output is generated by processing the downloaded data using the stat_v
     --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \
     --places_resolved_csv=place_resolver.csv
     ```
-

statvar_imports/nyu_diabetes/tennessee/download.py

@@ -13,16 +13,30 @@
 # limitations under the License.
 
 
+from datetime import date
 import os
-import requests
-from urllib.parse import urlparse
-from tqdm import tqdm  
-from retry import retry
 from pathlib import Path
-from datetime import date
-from absl import logging, app
-import pandas as pd
 import re
+from urllib.parse import urlparse
+
+from absl import app
+from absl import flags
+from absl import logging
+from google.api_core import exceptions
+from google.cloud import storage
+import pandas as pd
+import requests
+from retry import retry
+from tqdm import tqdm
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_enum(
+    'download_source',
+    'tn',
+    ['tn', 'gcs'],
+    'Source from which input files are downloaded.',
+)
 
 script_dir = os.path.dirname(os.path.abspath(__file__))
 INPUT_DIR = os.path.join(script_dir, "input_files")
@@ -43,7 +57,8 @@ def download_files(url_list, save_folder):
             filename = os.path.basename(parsed_url.path)
             file_path = os.path.join(save_folder, filename)
 
-            logging.info(f"Downloading: {filename}")
+            logging.info(
+                f"Starting download: source={url}, destination={file_path}")
 
             response = retry_method(url)
             with response as r:
@@ -55,14 +70,56 @@ def download_files(url_list, save_folder):
                     for chunk in r.iter_content(block_size):
                         f.write(chunk)
                         progress_bar.update(len(chunk))
-            logging.info(f"Saved: {file_path}\n")
+            file_size = os.path.getsize(file_path)
+            logging.info(
+                f"Completed download: source={url}, destination={file_path}, size_bytes={file_size}"
+            )
         except Exception as e:
-            logging.error(f"Failed to download {url} after retries: {e}\n")
+            logging.error(
+                f"Download failed: source={url}, destination={file_path}, error={e}"
+            )
+
+
+def download_files_from_gcs(url_list, save_folder):
+    os.makedirs(save_folder, exist_ok=True)
+    storage_client = storage.Client()
+    downloaded_count = 0
+
+    for url in url_list:
+        parsed_url = urlparse(url)
+        filename = os.path.basename(parsed_url.path)
+        file_path = os.path.join(save_folder, filename)
+        Path(file_path).unlink(missing_ok=True)
+
+        logging.info(
+            f"Starting download: source={url}, destination={file_path}")
+
+        try:
+            blob = storage_client.bucket(parsed_url.netloc).blob(
+                parsed_url.path.lstrip('/'))
+            blob.download_to_filename(file_path)
+        except exceptions.NotFound:
+            Path(file_path).unlink(missing_ok=True)
+            logging.warning(f"GCS object not found, skipping: source={url}")
+            continue
+        except Exception as e:
+            Path(file_path).unlink(missing_ok=True)
+            e.add_note(f"Failed to download GCS object {url}")
+            raise
+
+        file_size = os.path.getsize(file_path)
+        logging.info(
+            f"Completed download: source={url}, destination={file_path}, size_bytes={file_size}"
+        )
+        downloaded_count += 1
+
+    if downloaded_count == 0:
+        raise RuntimeError('No files were downloaded from GCS.')
 
 def generate_urls(start_year, end_year, url_template):
     url_list = []
     for year in range(start_year,end_year+1):
-        formatted_url = url_template.format(year,year)
+        formatted_url = url_template.format(year=year)
         url_list.append(formatted_url)
     return url_list
 
@@ -110,12 +167,17 @@ def process_excel_files(input_dir):
     logging.info("\nExcel file processing complete. 'year' column added to all processed files.")
 
 def main(_):
-    url_template ="https://www.tn.gov/content/dam/tn/health/documents/vital-statistics/death/{}/Diabetes_County_{}.xlsx"
+    tn_url_template = "https://www.tn.gov/content/dam/tn/health/documents/vital-statistics/death/{year}/Diabetes_County_{year}.xlsx"
+    gcs_url_template = "gs://unresolved_mcf/nyu_diabetes/tennessee/latest/input_files/Diabetes_County_{year}.xlsx"
     start_year = 2019
     current_year = date.today().year
-    final_urls = generate_urls(start_year, current_year,url_template)
-
-    download_files(final_urls, save_folder=INPUT_DIR)
+
+    if FLAGS.download_source == 'gcs':
+        final_urls = generate_urls(start_year, current_year, gcs_url_template)
+        download_files_from_gcs(final_urls, save_folder=INPUT_DIR)
+    else:
+        final_urls = generate_urls(start_year, current_year, tn_url_template)
+        download_files(final_urls, save_folder=INPUT_DIR)
     process_excel_files(INPUT_DIR)
 
 if __name__ == "__main__":

statvar_imports/nyu_diabetes/tennessee/download_test.py

@@ -0,0 +1,98 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+import tempfile
+import unittest
+from unittest import mock
+
+from google.api_core import exceptions
+
+import download
+
+
+class DownloadTest(unittest.TestCase):
+
+    def test_generate_urls_for_tn_and_gcs(self):
+        tn_template = 'https://tn.example/{year}/Diabetes_County_{year}.xlsx'
+        gcs_template = 'gs://test-bucket/input/Diabetes_County_{year}.xlsx'
+
+        self.assertEqual(
+            download.generate_urls(2019, 2020, tn_template),
+            [
+                'https://tn.example/2019/Diabetes_County_2019.xlsx',
+                'https://tn.example/2020/Diabetes_County_2020.xlsx',
+            ])
+        self.assertEqual(
+            download.generate_urls(2019, 2020, gcs_template),
+            [
+                'gs://test-bucket/input/Diabetes_County_2019.xlsx',
+                'gs://test-bucket/input/Diabetes_County_2020.xlsx',
+            ])
+
+    @mock.patch.object(download.storage, 'Client')
+    def test_gcs_download_skips_missing_file(self, mock_client):
+        downloaded_blob = mock.Mock()
+        downloaded_blob.download_to_filename.side_effect = (
+            lambda path: Path(path).write_bytes(b'new data'))
+        missing_blob = mock.Mock()
+        missing_blob.download_to_filename.side_effect = exceptions.NotFound(
+            'not found')
+        mock_client.return_value.bucket.return_value.blob.side_effect = [
+            downloaded_blob,
+            missing_blob,
+        ]
+        urls = [
+            'gs://test-bucket/input/Diabetes_County_2019.xlsx',
+            'gs://test-bucket/input/Diabetes_County_2020.xlsx',
+        ]
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            stale_file = Path(temp_dir) / 'Diabetes_County_2020.xlsx'
+            stale_file.write_bytes(b'stale data')
+
+            download.download_files_from_gcs(urls, temp_dir)
+
+            self.assertEqual(
+                (Path(temp_dir) / 'Diabetes_County_2019.xlsx').read_bytes(),
+                b'new data')
+            self.assertFalse(stale_file.exists())
+
+    @mock.patch.object(download.storage, 'Client')
+    def test_gcs_download_fails_when_no_files_exist(self, mock_client):
+        blob = mock_client.return_value.bucket.return_value.blob.return_value
+        blob.download_to_filename.side_effect = exceptions.NotFound('not found')
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            with self.assertRaisesRegex(RuntimeError,
+                                        'No files were downloaded from GCS'):
+                download.download_files_from_gcs(
+                    ['gs://test-bucket/input/Diabetes_County_2019.xlsx'],
+                    temp_dir)
+
+    @mock.patch.object(download.storage, 'Client')
+    def test_gcs_download_propagates_unexpected_error(self, mock_client):
+        blob = mock_client.return_value.bucket.return_value.blob.return_value
+        blob.download_to_filename.side_effect = exceptions.Forbidden(
+            'permission denied')
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            with self.assertRaises(exceptions.Forbidden):
+                download.download_files_from_gcs(
+                    ['gs://test-bucket/input/Diabetes_County_2019.xlsx'],
+                    temp_dir)
+
+
+if __name__ == '__main__':
+    unittest.main()

statvar_imports/nyu_diabetes/tennessee/manifest.json

@@ -5,7 +5,7 @@
       "curator_emails": ["support@datacommons.org"],
       "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html",
       "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.",
-      "scripts": ["download.py",
+      "scripts": ["download.py --download_source=gcs",
         "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output"
       ] ,
       "source_files": [
@@ -21,4 +21,3 @@
     }
   ]
 }
-