diff --git a/.gitignore b/.gitignore index 97fd024acf..ebe634aa2d 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,15 @@ err.txt # Oura OAuth app credentials (BYO; never committed) Strand/Oura/OuraSecrets.xcconfig + +# Pairing / ML large dumps (keep golden labels + status JSON only) +pairing-logs/noop-pairing-log.txt +pairing-logs/ml-samples.jsonl +pairing-logs/exports/**/*.jpg +pairing-logs/exports/**/*.jpeg +pairing-logs/exports/**/*.png +pairing-logs/datasets/ +pairing-logs/live/ +pairing-logs/whoop-apk/ +pairing-logs/*.db +pairing-logs/*.db-* diff --git a/Tools/calibrate_whoop_noop.py b/Tools/calibrate_whoop_noop.py new file mode 100644 index 0000000000..7d2d96ff7e --- /dev/null +++ b/Tools/calibrate_whoop_noop.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""CPU-only WHOOP↔NOOP affine calibration + honest metric table. + +Reads pairing-logs labels + daily features / optional noop day JSON. +Writes pairing-logs/calibration-report.json and prints a markdown table. + +No GPU. No synthetic WHOOP labels. +""" +from __future__ import annotations + +import argparse +import json +import math +import statistics +from datetime import date, datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +# Prefer in-repo pairing-logs; AI-store layout keeps them beside the checkout. +PAIRING = ROOT / "pairing-logs" +if not PAIRING.is_dir(): + PAIRING = ROOT.parent / "pairing-logs" +_ws = Path(r"C:\Users\Gilbert\Documents\Ai app store\pairing-logs") +if not PAIRING.is_dir() and _ws.is_dir(): + PAIRING = _ws + +LABELS = PAIRING / "whoop-app-labels.jsonl" +FEATURES = PAIRING / "ml-daily-features.json" +ASSETS = ROOT / "android" / "app" / "src" / "main" / "assets" / "whoop_app_labels.jsonl" +NOOP_DAYS = PAIRING / "noop-daily-metrics.jsonl" # optional export +OUT = PAIRING / "calibration-report.json" + +MIN_N_FIT = 3 +BAND = { + "charge": 12.0, + "effort": 15.0, + "sleep": 15.0, + "stress": 20.0, +} + + +def _load_jsonl(path: Path) -> list[dict]: + if not path.is_file(): + return [] + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + t = line.strip() + if not t or t.startswith("#"): + continue + try: + rows.append(json.loads(t)) + except json.JSONDecodeError: + continue + return rows + + +def _strain_to_100(v: float | None) -> float | None: + if v is None: + return None + if v <= 21.0 + 1e-6: + return max(0.0, min(100.0, v / 21.0 * 100.0)) + return max(0.0, min(100.0, v)) + + +def _effort_proxy_to_100(features_day: dict | None) -> float | None: + if not features_day: + return None + p = features_day.get("effort_proxy_0_100") + return float(p) if p is not None else None + + +def _affine_fit(xs: list[float], ys: list[float]) -> tuple[float, float]: + """Least-squares y ≈ a*x + b. Degenerate → identity.""" + n = len(xs) + if n < 2: + return 1.0, 0.0 + mx = statistics.mean(xs) + my = statistics.mean(ys) + varx = sum((x - mx) ** 2 for x in xs) + if varx < 1e-12: + return 1.0, my - mx + cov = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) + a = cov / varx + b = my - a * mx + return a, b + + +def _mae(pairs: list[tuple[float, float]]) -> float | None: + if not pairs: + return None + return sum(abs(a - b) for a, b in pairs) / len(pairs) + + +def _pearson(xs: list[float], ys: list[float]) -> float | None: + n = len(xs) + if n < 3: + return None + mx, my = statistics.mean(xs), statistics.mean(ys) + num = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) + denx = math.sqrt(sum((x - mx) ** 2 for x in xs)) + deny = math.sqrt(sum((y - my) ** 2 for y in ys)) + if denx < 1e-12 or deny < 1e-12: + return None + return num / (denx * deny) + + +def _loo_mae(xs: list[float], ys: list[float]) -> float | None: + n = len(xs) + if n < 3: + return None + errs = [] + for i in range(n): + tx = xs[:i] + xs[i + 1 :] + ty = ys[:i] + ys[i + 1 :] + a, b = _affine_fit(tx, ty) + pred = a * xs[i] + b + errs.append(abs(pred - ys[i])) + return sum(errs) / len(errs) + + +def collect_pairs(features: dict) -> dict[str, list[dict]]: + """Build per-head pairs from labels + optional noop metrics + feature proxies.""" + label_rows = _load_jsonl(LABELS) + _load_jsonl(ASSETS) + by_day: dict[str, dict] = {} + for r in label_rows: + day = r.get("day") + if not day: + continue + by_day.setdefault(day, {}).update( + { + "whoop_recovery": r.get("recovery_pct") if r.get("recovery_pct") is not None else r.get("recoveryPct"), + "whoop_strain_021": r.get("strain_021") if r.get("strain_021") is not None else r.get("dayStrain021"), + "whoop_sleep": r.get("sleep_pct") if r.get("sleep_pct") is not None else r.get("sleepPct"), + "whoop_stress": r.get("stress_pct") if r.get("stress_pct") is not None else r.get("stressPct"), + "source": r.get("source"), + } + ) + + for r in _load_jsonl(NOOP_DAYS): + day = r.get("day") + if not day: + continue + by_day.setdefault(day, {}).update( + { + "noop_recovery": r.get("recovery"), + "noop_strain": r.get("strain"), + "noop_sleep": r.get("sleep_performance") or r.get("rest"), + "noop_stress": r.get("stress_pct"), + } + ) + + feat_days = (features or {}).get("days") or {} + today = date.today().isoformat() + out = {"charge": [], "effort": [], "sleep": [], "stress": []} + for day, row in sorted(by_day.items()): + if day >= today: + continue # incomplete day guard + f = feat_days.get(day) + whoop_e = _strain_to_100(row.get("whoop_strain_021")) + noop_e = row.get("noop_strain") + if noop_e is None: + noop_e = _effort_proxy_to_100(f) + + if row.get("whoop_recovery") is not None and row.get("noop_recovery") is not None: + out["charge"].append( + { + "day": day, + "noop": float(row["noop_recovery"]), + "whoop": float(row["whoop_recovery"]), + } + ) + if whoop_e is not None and noop_e is not None: + out["effort"].append({"day": day, "noop": float(noop_e), "whoop": float(whoop_e)}) + if row.get("whoop_sleep") is not None and row.get("noop_sleep") is not None: + out["sleep"].append( + { + "day": day, + "noop": float(row["noop_sleep"]), + "whoop": float(row["whoop_sleep"]), + } + ) + if row.get("whoop_stress") is not None and row.get("noop_stress") is not None: + out["stress"].append( + { + "day": day, + "noop": float(row["noop_stress"]), + "whoop": float(row["whoop_stress"]), + } + ) + return out + + +def evaluate_head(name: str, pairs: list[dict]) -> dict: + xs = [p["noop"] for p in pairs] + ys = [p["whoop"] for p in pairs] + raw = list(zip(xs, ys)) + mae_before = _mae(raw) + a, b = (1.0, 0.0) + mae_after = mae_before + loo = None + r = _pearson(xs, ys) + fitted = False + if len(pairs) >= MIN_N_FIT: + a, b = _affine_fit(xs, ys) + fitted = True + cal = [(a * x + b, y) for x, y in raw] + mae_after = _mae(cal) + loo = _loo_mae(xs, ys) + within = None + if pairs: + band = BAND[name] + within = sum(1 for x, y in raw if abs(x - y) <= band) / len(raw) + return { + "head": name, + "n": len(pairs), + "mae_before": mae_before, + "mae_after_affine": mae_after, + "loo_mae": loo, + "pearson_r": r, + "affine_a": a, + "affine_b": b, + "fitted": fitted, + "within_band_frac": within, + "band": BAND[name], + "days": [p["day"] for p in pairs], + "citations": "Plews2013 / Banister1991 / Cole-Kripke1992 / Baevsky2008; affine per docs/CALIBRATION.md", + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--pairing", type=Path, default=PAIRING) + args = ap.parse_args() + pairing = args.pairing + features = {} + feat_path = pairing / "ml-daily-features.json" + if feat_path.is_file(): + features = json.loads(feat_path.read_text(encoding="utf-8")) + + global LABELS, ASSETS, NOOP_DAYS, OUT + LABELS = pairing / "whoop-app-labels.jsonl" + NOOP_DAYS = pairing / "noop-daily-metrics.jsonl" + OUT = pairing / "calibration-report.json" + + pairs = collect_pairs(features) + heads = [evaluate_head(k, pairs[k]) for k in ("charge", "effort", "sleep", "stress")] + n_fit = sum(1 for h in heads if h["fitted"]) + n_any = sum(h["n"] for h in heads) + accuracy_valid = n_fit >= 2 and all( + h["n"] >= MIN_N_FIT or h["n"] == 0 for h in heads if h["head"] in ("charge", "effort", "sleep") + ) + # Stricter: Charge + Effort + Sleep must each have n>=3 to claim valid multi-head accuracy. + accuracy_valid = all(h["n"] >= MIN_N_FIT for h in heads if h["head"] in ("charge", "effort", "sleep")) + + report = { + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "accuracy_valid": accuracy_valid, + "min_n_fit": MIN_N_FIT, + "n_label_rows": len(_load_jsonl(LABELS) + _load_jsonl(ASSETS)), + "n_pairs_total": n_any, + "heads_fitted": n_fit, + "gpu": "none (CPU-only)", + "method": "affine least-squares + LOO MAE; shared 0-100 (strain ×100/21)", + "citations_doc": "docs/CALIBRATION.md", + "heads": heads, + "deploy_gate": "PASS" if accuracy_valid else "FAIL - need >=3 paired days on Charge, Effort, Sleep", + "note": "Never invent WHOOP labels. Sparse N means gate fail, not fake high accuracy.", + } + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(json.dumps(report, indent=2), encoding="utf-8") + + def _fmt(v): + return "-" if v is None else f"{v:.2f}" if isinstance(v, float) else str(v) + + print("| Head | N | MAE before | MAE after affine | Pearson r | Fitted |") + print("|------|---|------------|------------------|-----------|--------|") + for h in heads: + pr = "-" if h["pearson_r"] is None else f"{h['pearson_r']:.3f}" + print( + f"| {h['head']} | {h['n']} | {_fmt(h['mae_before'])} | " + f"{_fmt(h['mae_after_affine'])} | {pr} | {h['fitted']} |" + ) + print() + print(f"accuracy_valid={accuracy_valid} deploy_gate={report['deploy_gate']}") + print(f"wrote {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Tools/ingest_export.ps1 b/Tools/ingest_export.ps1 new file mode 100644 index 0000000000..c1bac2407c --- /dev/null +++ b/Tools/ingest_export.ps1 @@ -0,0 +1,204 @@ +# Ingest a WHOOP/NOOP screenshot export from Downloads into pairing-logs/exports/ +# and emit a manifest.csv that pairs same-minute WHOOP<->NOOP captures. +# +# Usage: +# powershell -File Tools\ingest_export.ps1 +# powershell -File Tools\ingest_export.ps1 -ExportDir "C:\Users\Gilbert\Downloads\Noop mg-...\Noop mg" +# powershell -File Tools\ingest_export.ps1 -ExportDir "...\Noop stresd" -PackKind stress +# +# Filenames are the source of truth: Screenshot_YYYYMMDD_HHMMSS_.jpg +# (Samsung stamps capture time + foreground app). The manifest adds empty +# `screen` / `values` columns that the decode pass fills in, so the CSV — +# not prose — becomes the durable record for that export. +# +# Screen taxonomy + three-lane compare: +# docs/WHOOP_NOOP_SCREENSHOT_COMPARE.md (AI app store root) + +param( + [string]$ExportDir = "", + [int]$PairWindowMinutes = 10, + [ValidateSet("auto", "stress", "sleep", "mixed")] + [string]$PackKind = "auto" +) + +$ErrorActionPreference = "Stop" +$downloads = Join-Path $env:USERPROFILE "Downloads" + +# Prefer repo pairing-logs/exports (noop checkout). Fall back to AI-store workspace. +$exportsRoot = Join-Path $PSScriptRoot "..\pairing-logs\exports" +$wsExports = "C:\Users\Gilbert\Documents\Ai app store\pairing-logs\exports" +if (-not (Test-Path (Split-Path $exportsRoot)) -and (Test-Path (Split-Path $wsExports))) { + $exportsRoot = $wsExports +} + +$pattern = '^Screenshot_(\d{8})_(\d{6})_(.+)\.(jpg|jpeg|png)$' + +function Infer-PackKind([object[]]$rows) { + $names = ($rows | ForEach-Object { $_.file }) -join " " + $stressHints = 0; $sleepHints = 0 + # Filename alone rarely encodes screen — default mixed; caller should pass -PackKind + if ($PackKind -ne "auto") { return $PackKind } + return "mixed" +} + +$requiredByKind = @{ + stress = @( + @{ Id = "S1"; App = "WHOOP"; Screen = "whoop_home"; Why = "Home Stress Monitor card (tip+band)" }, + @{ Id = "S2"; App = "WHOOP"; Screen = "whoop_stress_monitor"; Why = "Full Stress Monitor chart + high-zone copy" }, + @{ Id = "S3"; App = "NOOP"; Screen = "noop_today_health"; Why = "Today Health/Key Metrics stress row" }, + @{ Id = "S4"; App = "NOOP"; Screen = "noop_stress_hero"; Why = "Stress hero Now tip" }, + @{ Id = "S5"; App = "NOOP"; Screen = "noop_stress_timeline"; Why = "Intraday timeline + time-in-band (do not skip)" } + ) + sleep = @( + @{ Id = "L1"; App = "WHOOP"; Screen = "whoop_home"; Why = "Home rings + sleep activity" }, + @{ Id = "L2"; App = "WHOOP"; Screen = "whoop_sleep_detail"; Why = "Hours of sleep + stage bars" }, + @{ Id = "L3"; App = "NOOP"; Screen = "noop_sleep_hero"; Why = "Rest gauge + What shaped Rest" }, + @{ Id = "L4"; App = "NOOP"; Screen = "noop_sleep_stages"; Why = "Stage minutes / honesty strip" }, + @{ Id = "L5"; App = "NOOP"; Screen = "noop_today_health"; Why = "Today Rest vessel same wake-day" } + ) + mixed = @( + @{ Id = "M1"; App = "WHOOP"; Screen = "whoop_home"; Why = "Home rings + Stress card" }, + @{ Id = "M2"; App = "WHOOP"; Screen = "whoop_stress_monitor"; Why = "Stress Monitor if comparing stress" }, + @{ Id = "M3"; App = "WHOOP"; Screen = "whoop_sleep_detail"; Why = "Sleep detail if comparing Rest" }, + @{ Id = "M4"; App = "NOOP"; Screen = "noop_today_health"; Why = "Today Health row" }, + @{ Id = "M5"; App = "NOOP"; Screen = "noop_stress_timeline"; Why = "NOOP stress chart (often missing)" }, + @{ Id = "M6"; App = "NOOP"; Screen = "noop_sleep_hero"; Why = "Sleep Rest hero" } + ) +} + +if (-not $ExportDir) { + $candidates = Get-ChildItem $downloads -Directory | ForEach-Object { + $leafDirs = @($_) + (Get-ChildItem $_.FullName -Directory -ErrorAction SilentlyContinue) + foreach ($d in $leafDirs) { + $shots = Get-ChildItem $d.FullName -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -match $pattern } + if ($shots) { + [pscustomobject]@{ Dir = $d.FullName; Newest = ($shots | Sort-Object Name | Select-Object -Last 1).Name } + } + } + } + if (-not $candidates) { throw "No export folder with Screenshot_*_APP.jpg files found under $downloads" } + $ExportDir = ($candidates | Sort-Object Newest | Select-Object -Last 1).Dir +} + +Write-Host "Ingesting: $ExportDir" + +$shots = @(Get-ChildItem $ExportDir -File | Where-Object { $_.Name -match $pattern }) +if (-not $shots) { throw "No Screenshot_*_APP files in $ExportDir" } + +$rows = foreach ($f in $shots) { + $null = $f.Name -match $pattern + $d = $Matches[1]; $t = $Matches[2]; $app = $Matches[3].ToUpper() + # Normalize package-style suffixes (com.whoop... etc.) to WHOOP/NOOP when obvious + if ($app -match 'WHOOP') { $app = 'WHOOP' } + elseif ($app -match 'NOOP|STRAND') { $app = 'NOOP' } + [pscustomobject]@{ + file = $f.Name + app = $app + captured = [datetime]::ParseExact("$d$t", "yyyyMMddHHmmss", $null) + screen = "" + values = "" + pairFile = "" + pairGapS = "" + } +} +$rows = @($rows | Sort-Object captured) +$kind = Infer-PackKind $rows + +# pair each capture with the nearest capture from the OTHER app within the window +foreach ($r in $rows) { + $others = $rows | Where-Object { $_.app -ne $r.app } + if ($others) { + $best = $others | Sort-Object { [math]::Abs(($_.captured - $r.captured).TotalSeconds) } | Select-Object -First 1 + $gap = [math]::Abs(($best.captured - $r.captured).TotalSeconds) + if ($gap -le $PairWindowMinutes * 60) { + $r.pairFile = $best.file + $r.pairGapS = [int]$gap + } + } +} + +$stamp = ($rows[0].captured).ToString("yyyyMMdd") + "-" + (Split-Path $ExportDir -Leaf).Trim() -replace '[^\w\-]', '_' +$dest = Join-Path $exportsRoot $stamp +New-Item -ItemType Directory -Force $dest | Out-Null +foreach ($f in $shots) { Copy-Item $f.FullName (Join-Path $dest $f.Name) -Force } + +$manifest = Join-Path $dest "manifest.csv" +$rows | Select-Object file, app, @{n='captured';e={$_.captured.ToString("yyyy-MM-dd HH:mm:ss")}}, screen, values, pairFile, pairGapS | + Export-Csv $manifest -NoTypeInformation -Encoding UTF8 + +# REQUIRED_SHOTS.md - checklist vs inventory (screen still empty until decode) +$req = $requiredByKind[$kind] +$reqPath = Join-Path $dest "REQUIRED_SHOTS.md" +$sb = New-Object System.Text.StringBuilder +[void]$sb.AppendLine("# Required shots - pack kind $kind") +[void]$sb.AppendLine("") +[void]$sb.AppendLine("Playbook: docs/WHOOP_NOOP_SCREENSHOT_COMPARE.md") +[void]$sb.AppendLine("Export: $ExportDir") +[void]$sb.AppendLine("Copied to: $dest") +[void]$sb.AppendLine("") +[void]$sb.AppendLine("| Id | Need | App | Target screen | Status |") +[void]$sb.AppendLine("|----|------|-----|---------------|--------|") +foreach ($item in $req) { + $haveApp = @($rows | Where-Object { $_.app -eq $item.App }).Count + if ($haveApp -gt 0) { + $status = "HAVE $($item.App) files ($haveApp) - assign screen=$($item.Screen) in manifest during decode" + } else { + $status = "MISSING any $($item.App) shot" + } + [void]$sb.AppendLine("| $($item.Id) | $($item.Why) | $($item.App) | $($item.Screen) | $status |") +} +[void]$sb.AppendLine("") +[void]$sb.AppendLine("After decode: mark each Id DONE only when a row has that screen value filled.") +[void]$sb.AppendLine("Stress packs: S5 (noop_stress_timeline) was missing in Noop-mg 2026-07-14 - always capture it.") +[System.IO.File]::WriteAllText($reqPath, $sb.ToString()) + +# decode_worksheet.md - one block per file for Read-tool decode passes +$wsPath = Join-Path $dest "decode_worksheet.md" +$wsb = New-Object System.Text.StringBuilder +[void]$wsb.AppendLine("# Decode worksheet") +[void]$wsb.AppendLine("") +[void]$wsb.AppendLine("Fill screen + values here, then copy into manifest.csv.") +[void]$wsb.AppendLine("Taxonomy keys: docs/WHOOP_NOOP_SCREENSHOT_COMPARE.md sections 3-4.") +[void]$wsb.AppendLine("") +foreach ($r in $rows) { + $pair = if ($r.pairFile) { "$($r.pairFile) (gap $($r.pairGapS)s)" } else { "(unpaired)" } + [void]$wsb.AppendLine("## $($r.file)") + [void]$wsb.AppendLine("") + [void]$wsb.AppendLine("- app: $($r.app) | captured: $($r.captured.ToString('yyyy-MM-dd HH:mm:ss')) | pair: $pair") + [void]$wsb.AppendLine("- screen: (whoop_stress_monitor / noop_stress_timeline / ...)") + [void]$wsb.AppendLine("- values: (tip=; band=; tipClock=; highZone=; statusBarClock=; ...)") + [void]$wsb.AppendLine("- chart notes: (night floor / peak hour / activity glyph / missing series)") + [void]$wsb.AppendLine("") +} +[System.IO.File]::WriteAllText($wsPath, $wsb.ToString()) + +# labels stub template (append manually after decode - do not auto-write guessing) +$stubPath = Join-Path $dest "labels_stub.jsonl.example" +$stubLines = foreach ($r in @($rows | Where-Object { $_.app -eq 'WHOOP' -and $_.pairFile })) { + $day = $r.captured.ToString("yyyy-MM-dd") + $clock = $r.captured.ToString("HH:mm") + '{"day":"' + $day + '","source":"screenshot","clock":"' + $clock + '","whoop_file":"' + $r.file + '","noop_file":"' + $r.pairFile + '","stress_tip":null,"stress_band":null,"noop_tip":null,"serial":"export:' + $stamp + '"}' +} +if ($stubLines) { + [System.IO.File]::WriteAllLines($stubPath, @($stubLines)) +} + +$byApp = $rows | Group-Object app | ForEach-Object { "$($_.Name)=$($_.Count)" } +$span = "{0:HH:mm:ss} - {1:HH:mm:ss}" -f $rows[0].captured, $rows[-1].captured +$paired = @($rows | Where-Object { $_.pairGapS -ne "" }).Count +$tight = @($rows | Where-Object { $_.pairGapS -ne "" -and [int]$_.pairGapS -le 120 }).Count + +Write-Host "" +Write-Host "Copied $($rows.Count) shots -> $dest" +Write-Host "PackKind: $kind Apps: $($byApp -join ', ') Span: $span" +Write-Host "Paired (<= ${PairWindowMinutes}m): $paired/$($rows.Count) Tight (<=2m tip@clock): $tight" +Write-Host "Manifest: $manifest" +Write-Host "Checklist: $reqPath" +Write-Host "Worksheet: $wsPath" +if (Test-Path $stubPath) { Write-Host "Labels: $stubPath (fill tips, append to pairing-logs\whoop-app-labels.jsonl)" } +Write-Host "" +Write-Host "Next:" +Write-Host " 1. Read each JPG; fill screen+values in manifest.csv (worksheet helps)." +Write-Host " 2. Tip@clock table + three-lane compare per docs\WHOOP_NOOP_SCREENSHOT_COMPARE.md" +Write-Host " 3. Update factor docs; ANY_MODEL_CONTINUE.md last." diff --git a/Tools/ml_engine_train.py b/Tools/ml_engine_train.py new file mode 100644 index 0000000000..e24544d516 --- /dev/null +++ b/Tools/ml_engine_train.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +"""NOOP ML engine — train/eval Effort toward WHOOP **app** Strain labels. + +Accuracy is ONLY reported when real labels exist (whoop-app dumps / manual JSON / export). +Synthetic or empty labels → status accuracy_valid=false (never claim %). + +Pipeline always builds a daily feature store from ML_SAMPLE / log so the moment labels +arrive, fitting is ready (G5). Loops should read pairing-logs/ml-engine-status.json and +update Goals (G4/G5), not invent pass scores from pipeline-only HR. + +Usage: + python Tools/ml_engine_train.py + python Tools/ml_engine_train.py --labels pairing-logs/whoop-app-labels.jsonl +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import statistics +from collections import defaultdict +from datetime import date, datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "pairing-logs" / "ml-engine-status.json" +WEIGHTS = ROOT / "pairing-logs" / "ml-effort-weights.json" +FEATURES_OUT = ROOT / "pairing-logs" / "ml-daily-features.json" +SPORT_FEATURES_OUT = ROOT / "pairing-logs" / "ml-sport-session-features.json" +GOALS_OUT = ROOT / "pairing-logs" / "goals-from-ml.json" +LOG = ROOT / "pairing-logs" / "noop-pairing-log.txt" +SAMPLES = ROOT / "pairing-logs" / "ml-samples.jsonl" +DUMPS = ROOT / "pairing-logs" / "whoop-app-dumps" +LABELS_JSONL = ROOT / "pairing-logs" / "whoop-app-labels.jsonl" + + +def _read_json_text(path: Path) -> str: + """Read text accepting UTF-8 BOM (PowerShell Set-Content -Encoding utf8).""" + return path.read_text(encoding="utf-8-sig", errors="replace") + + +def load_labels(path: Path) -> list[dict]: + rows: list[dict] = [] + if path.exists(): + for line in _read_json_text(path).splitlines(): + line = line.strip().lstrip("\ufeff") + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + if DUMPS.is_dir(): + for p in sorted(DUMPS.glob("scores-*.json")): + try: + d = json.loads(_read_json_text(p)) + if d.get("day_strain_021") is not None or d.get("recovery_pct") is not None: + rows.append( + { + "day": d.get("day") or p.stem, + "strain_021": d.get("day_strain_021"), + "recovery_pct": d.get("recovery_pct"), + "source": d.get("source") or "adb_dump", + } + ) + except Exception: + pass + # Dedupe by day: keep richest row (prefer recovery present, then later in list) + by_day: dict[str, dict] = {} + for r in rows: + day = str(r.get("day") or r.get("day_raw") or "").strip() + if not day or day.startswith("scores-"): + # keep undated rows with synthetic keys so they don't collapse + day = f"_undated_{len(by_day)}_{r.get('source')}" + prev = by_day.get(day) + if prev is None: + by_day[day] = r + continue + prev_score = (1 if prev.get("recovery_pct") is not None else 0) + ( + 1 if prev.get("strain_021") is not None else 0 + ) + new_score = (1 if r.get("recovery_pct") is not None else 0) + ( + 1 if r.get("strain_021") is not None else 0 + ) + if new_score >= prev_score: + by_day[day] = r + return list(by_day.values()) + + +def _day_key_from_recv(recv_at: str | None, ts_ms: int | None) -> str | None: + if recv_at: + # 2026-07-10T18:41:15+00:00 + try: + return recv_at[:10] + except Exception: + pass + if ts_ms and ts_ms > 1_000_000_000_000: + try: + return datetime.fromtimestamp(ts_ms / 1000.0, tz=timezone.utc).strftime("%Y-%m-%d") + except Exception: + pass + return None + + +def load_ml_samples() -> list[dict]: + rows = [] + if SAMPLES.exists(): + for line in SAMPLES.read_text(encoding="utf-8", errors="replace").splitlines(): + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + # Also scrape last ~1.5MB of pairing log for ML_SAMPLE lines (richer R-R sometimes) + if LOG.exists(): + data = LOG.read_bytes()[-1_500_000:].decode("utf-8", "replace") + for m in re.finditer( + r"ML_SAMPLE[^\n]*?(?:ts_ms=(\d+))?[^\n]*?hr=(\d{2,3})(?:[^\n]*?rr=\[([^\]]*)\])?", + data, + ): + ts_s, hr_s, rr_s = m.group(1), m.group(2), m.group(3) + # Try to get ISO prefix on same line + line_start = data.rfind("\n", 0, m.start()) + 1 + line = data[line_start : m.end()] + day = None + iso = re.match(r"(\d{4}-\d{2}-\d{2})", line) + if iso: + day = iso.group(1) + rr = [] + if rr_s: + rr = [int(x) for x in re.findall(r"\d+", rr_s)] + rows.append( + { + "kind": "ML_SAMPLE", + "ts_ms": int(ts_s) if ts_s else None, + "hr": int(hr_s), + "rr": rr, + "day": day, + "source": "log_scrape", + } + ) + return rows + + +def load_sport_labels(samples: list[dict]) -> list[dict]: + """Load debug-only, user-confirmed workout labels without mixing them into daily WHOOP labels.""" + rows = [s for s in samples if s.get("kind") == "ML_WORKOUT_LABEL"] + if LOG.exists(): + data = LOG.read_bytes()[-1_500_000:].decode("utf-8", "replace") + pattern = re.compile( + r"ML_WORKOUT_LABEL\s+v=(?P\d+)\s+label_ts_ms=(?P