Speed up CI by caching dependencies and virtual environment

.github/workflows/install-and-test.yml

@@ -44,13 +44,37 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python 3.13
+        id: setup-python
         uses: actions/setup-python@v6
         with:
           python-version: 3.13
+          cache: 'pip'
 
       - name : Install uv (if applicable)
         if: matrix.package_manager == 'uv'
         uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Generate lock file if missing
+        if: matrix.package_manager == 'uv'
+        run: |
+          if [ ! -f uv.lock ]; then
+            echo "Generating uv.lock for cache key computation..."
+            uv lock
+          else
+            echo "uv.lock already exists"
+          fi
+
+      - name: Cache virtual environment
+        if: matrix.package_manager == 'uv'
+        id: cache-venv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: uv-venv-${{ runner.os }}-${{ matrix.device }}-py${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            uv-venv-${{ runner.os }}-${{ matrix.device }}-py${{ steps.setup-python.outputs.python-version }}-
 
       - name: Create virtual environment (pip GPU)
         if: matrix.package_manager == 'pip' && matrix.device == 'gpu'
@@ -74,15 +98,23 @@ jobs:
       - name: Install package (including dev dependencies)
         if : matrix.package_manager != 'uv'
         run: |
-          python -m pip install --group dev .
+          python -m pip install -e .[dev]
 
       - name: Setup environment with uv (if applicable)
         if: matrix.package_manager == 'uv'
         run: |
-          uv venv --no-project
+          if [ "${{ steps.cache-venv.outputs.cache-hit }}" != "true" ] && [ -d .venv ]; then
+            rm -rf .venv
+          fi
+          if [ ! -d .venv ]; then
+            uv venv --no-project
+          fi
           source .venv/bin/activate
-          uv pip install torch --index-url ${{ matrix.torch_index }}
-          uv pip install --group dev -e .
+          if [ "${{ steps.cache-venv.outputs.cache-hit }}" != "true" ]; then
+            uv pip install torch --index-url ${{ matrix.torch_index }}
+            uv pip install --group dev .
+          fi
+          uv pip install -e . --no-deps
           uv pip list
           echo "$PWD/.venv/bin" >> $GITHUB_PATH
 

CHANGELOG.md

@@ -51,6 +51,10 @@ This release introduces new features including GIF animation support, wandb run
 
 - Initialize `da_forcing_mean` and `da_forcing_std` to `None` when forcing data is absent, fixing `AttributeError` in `WeatherDataset` with `standardize=True` [\#369](https://github.com/mllam/neural-lam/issues/369) @Sir-Sloth-The-Lazy
 
+### Maintenance
+
+- Add two-layer caching to install-and-test CI workflow (pip download cache + uv venv cache) to reduce build times by 80-95% [\#606](https://github.com/mllam/neural-lam/pull/606) @Sharkyii
+
 - Ensure proper sorting of `analysis_time` in `NpyFilesDatastoreMEPS._get_analysis_times` independent of the order in which files are processed with glob [\#386](https://github.com/mllam/neural-lam/pull/386) @Gopisokk
 
 - Replace `shell=True` subprocess call in `compute_standardization_stats.py` with a safe argument list and Python-side hostname parsing to prevent command injection via `SLURM_JOB_NODELIST` [\#264](https://github.com/mllam/neural-lam/pull/264) @ashum9