Merge pull request #202 from CenterForTheBuiltEnvironment/testing-scheme-v2

status: add full testing scheme v2

Author: t-kramer

.github/workflows/deploy.yml

@@ -24,9 +24,8 @@ jobs:
       - name: Install dependencies
         run: pip install -r requirements.txt
 
-      - name: Run tests
-        run: pytest
-        continue-on-error: true  # TODO: remove once test suite is complete
+      - name: Run full test suite (all tiers; gates deployment)
+        run: python -m pytest -q --tb=short
 
   deploy:
     name: Build & deploy to Cloud Run

.github/workflows/tests.yml

@@ -0,0 +1,26 @@
+name: Tests
+
+# Full suite on feature-branch pushes and PRs into development.
+# Pushes/PRs to main run the same suite in deploy.yml, which gates deployment.
+on:
+  push:
+    branches-ignore: [main]
+  pull_request:
+    branches: [development]
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: pip
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run full test suite
+        run: python -m pytest -q --tb=short

.gitignore

@@ -31,4 +31,7 @@ test-measured-data.ipynb
 
 # AI assistant logs
 .claude/
-CLAUDE.md
+CLAUDE.md
+
+# Local hand-calculation spreadsheet (frozen into tests/fixtures/spreadsheet_cases.json)
+tests/testing_cases.xlsx

.pre-commit-config.yaml

@@ -5,3 +5,12 @@ repos:
       - id: ruff
         args: [--fix]
       - id: ruff-format
+
+  - repo: local
+    hooks:
+      - id: pytest-unit
+        name: pytest (unit tier)
+        entry: python -m pytest -m unit -x -q --tb=short
+        language: system
+        pass_filenames: false
+        stages: [pre-commit]

notes/testing-scheme.md

@@ -34,4 +34,11 @@ indent-style = "space"
 testpaths = ["tests"]
 python_files = ["test_*.py"]
 python_functions = ["test_*"]
+markers = [
+  "unit: fast pure-function tests, synthetic data, no I/O (<5s total)",
+  "verification: engine + emissions checked against independent hand calculations and an independent reference model",
+  "regression: physics invariants + syrupy snapshots using the real equipment library",
+  "integration: full 8760-hour runs on real load and Cambium data, golden annual totals (1-3 min)",
+  "app: Dash server smoke tests and headless end-to-end workflow",
+]
 addopts = ["-v", "--tb=short", "-ra"]

pyproject.toml

@@ -79,6 +79,15 @@ def _validate(df: pd.DataFrame) -> pd.DataFrame:
             if bad_count > 0:
                 logger.warning(f"{bad_count} invalid values in column {col}, set to NaN")
 
+        # Reject negative load values — a negative heating or cooling load indicates
+        # a data error (unit mismatch, sign convention issue, etc.)
+        for col in ["heating_W", "cooling_W"]:
+            if (df[col].dropna() < 0).any():
+                raise ValueError(
+                    f"Column '{col}' contains negative values. "
+                    "Load values must be >= 0. Check for unit errors or sign convention issues."
+                )
+
         return df
 
     # --------- Factory methods ---------

src/loads.py

@@ -0,0 +1,72 @@
+"""Helpers for the app-tier tests: walk Dash layouts without a browser."""
+
+import json
+import re
+from pathlib import Path
+
+from dash.development.base_component import Component
+
+ROOT = Path(__file__).resolve().parents[1]
+_ID_IN_SOURCE = re.compile(r"""\bid\s*=\s*["']([\w\-:.]+)["']""")
+
+
+def walk(node):
+    """Yield every Dash component in a layout tree (children and component-valued props)."""
+    if isinstance(node, Component):
+        yield node
+        for prop in node._prop_names:
+            value = getattr(node, prop, None)
+            if isinstance(value, Component | list | tuple):
+                yield from walk(value)
+    elif isinstance(node, list | tuple):
+        for item in node:
+            yield from walk(item)
+
+
+def component_ids(layout) -> set[str]:
+    ids = set()
+    for comp in walk(layout):
+        cid = getattr(comp, "id", None)
+        if isinstance(cid, str):
+            ids.add(cid)
+    return ids
+
+
+def find(layout, component_id: str) -> Component:
+    for comp in walk(layout):
+        if getattr(comp, "id", None) == component_id:
+            return comp
+    raise KeyError(component_id)
+
+
+def option_values(layout, component_id: str) -> list:
+    """Values of a Select/SegmentedControl's `data` options, read from the live layout."""
+    return [opt["value"] for opt in find(layout, component_id).data]
+
+
+def ids_declared_in_source() -> set[str]:
+    """Every literal id="..." in the app source, including components built at runtime."""
+    ids = set()
+    for folder in ("pages", "layout"):
+        for path in (ROOT / folder).glob("*.py"):
+            ids.update(_ID_IN_SOURCE.findall(path.read_text()))
+    ids.update(_ID_IN_SOURCE.findall((ROOT / "app.py").read_text()))
+    return ids
+
+
+def callback_component_ids(dependencies: list[dict]) -> set[str]:
+    """String component ids referenced by /_dash-dependencies (pattern-matching ids skipped)."""
+    ids = set()
+    for dep in dependencies:
+        if not dep.get("no_output"):
+            for output in dep["output"].strip(".").split("..."):
+                ids.add(output.rsplit(".", 1)[0])
+        for item in dep["inputs"] + dep["state"]:
+            if isinstance(item["id"], str):
+                ids.add(item["id"])
+    return {i for i in ids if not i.startswith("{")}
+
+
+def figure_has_data(fig) -> bool:
+    fig = json.loads(fig.to_json()) if hasattr(fig, "to_json") else fig
+    return any(trace for trace in fig.get("data", []))

tests/__snapshots__/test_energy_regression.ambr

@@ -10,6 +10,33 @@
 from src.loads import StandardLoad
 from src.metadata import Metadata
 
+TIER_MARKERS = {"unit", "verification", "regression", "integration", "app"}
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--generate-golden",
+        action="store_true",
+        default=False,
+        help="Regenerate integration_annual_totals.json golden values (integration tier)",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    """Fail collection if any test lacks a tier marker.
+
+    CI selects tests by marker (e.g. -m "unit or regression"), so an unmarked
+    test would silently never run there.
+    """
+    unmarked = [
+        item.nodeid for item in items if not TIER_MARKERS & {m.name for m in item.iter_markers()}
+    ]
+    if unmarked:
+        raise pytest.UsageError(
+            f"Tests without a tier marker ({', '.join(sorted(TIER_MARKERS))}):\n  "
+            + "\n  ".join(unmarked)
+        )
+
 
 @pytest.fixture
 def sample_load_df():