From 0c02115190e103636d30be4716e03b9603651d0d Mon Sep 17 00:00:00 2001 From: Derek Clair Date: Wed, 19 Aug 2026 22:28:21 -0600 Subject: [PATCH] Initial public-ready snapshot of the LangGraph memory-aware agent brain. Pluggable Grok / Claude / local NIM providers, proactive Supermemory injection, and a small test suite. Companion to conversational-voice-agent. --- .env.example | 33 +++ .github/workflows/ci.yml | 42 +++ .gitignore | 165 ++++++++++++ Dockerfile | 45 ++++ LICENSE | 21 ++ Makefile | 184 +++++++++++++ README.md | 215 ++++++++++++++- benchmarks/README.md | 70 +++++ benchmarks/__init__.py | 16 ++ benchmarks/metrics.py | 270 +++++++++++++++++++ benchmarks/reports/.gitkeep | 0 benchmarks/runner.py | 218 +++++++++++++++ docker-compose.yml | 132 +++++++++ docs/architecture.md | 63 +++++ docs/development.md | 98 +++++++ examples/basic_chat.py | 31 +++ examples/official_guide_style.py | 42 +++ pyproject.toml | 86 ++++++ specs/001-voice-dgx-spark-agent/plan.md | 241 +++++++++++++++++ specs/001-voice-dgx-spark-agent/spec.md | 162 +++++++++++ specs/001-voice-dgx-spark-agent/tasks.md | 199 ++++++++++++++ specs/002-multi-user-support/spec.md | 98 +++++++ specs/003-deployment-infrastructure/spec.md | 109 ++++++++ specs/004-persistence-checkpointers/spec.md | 65 +++++ specs/005-testing-and-cicd/spec.md | 92 +++++++ specs/006-alternative-memory-systems/spec.md | 61 +++++ specs/007-dgx-hardware-optimization/plan.md | 152 +++++++++++ specs/007-dgx-hardware-optimization/spec.md | 220 +++++++++++++++ specs/007-dgx-hardware-optimization/tasks.md | 144 ++++++++++ src/thelab_langchain/__init__.py | 28 ++ src/thelab_langchain/agent/__init__.py | 0 src/thelab_langchain/agent/graph.py | 137 ++++++++++ src/thelab_langchain/agent/state.py | 32 +++ src/thelab_langchain/agent/tools/__init__.py | 0 src/thelab_langchain/agent/tools/memory.py | 108 ++++++++ src/thelab_langchain/chat.py | 181 +++++++++++++ src/thelab_langchain/cli.py | 201 ++++++++++++++ src/thelab_langchain/config.py | 106 ++++++++ src/thelab_langchain/llm.py | 62 +++++ src/thelab_langchain/voice/__init__.py | 24 ++ src/thelab_langchain/voice/audio.py | 114 ++++++++ src/thelab_langchain/voice/orchestrator.py | 208 ++++++++++++++ src/thelab_langchain/voice/riva.py | 125 +++++++++ tests/__init__.py | 0 tests/test_agent_graph.py | 107 ++++++++ tests/test_chat.py | 29 ++ tests/test_config.py | 60 +++++ tests/test_llm.py | 68 +++++ 48 files changed, 4862 insertions(+), 2 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 benchmarks/README.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/metrics.py create mode 100644 benchmarks/reports/.gitkeep create mode 100644 benchmarks/runner.py create mode 100644 docker-compose.yml create mode 100644 docs/architecture.md create mode 100644 docs/development.md create mode 100644 examples/basic_chat.py create mode 100644 examples/official_guide_style.py create mode 100644 pyproject.toml create mode 100644 specs/001-voice-dgx-spark-agent/plan.md create mode 100644 specs/001-voice-dgx-spark-agent/spec.md create mode 100644 specs/001-voice-dgx-spark-agent/tasks.md create mode 100644 specs/002-multi-user-support/spec.md create mode 100644 specs/003-deployment-infrastructure/spec.md create mode 100644 specs/004-persistence-checkpointers/spec.md create mode 100644 specs/005-testing-and-cicd/spec.md create mode 100644 specs/006-alternative-memory-systems/spec.md create mode 100644 specs/007-dgx-hardware-optimization/plan.md create mode 100644 specs/007-dgx-hardware-optimization/spec.md create mode 100644 specs/007-dgx-hardware-optimization/tasks.md create mode 100644 src/thelab_langchain/__init__.py create mode 100644 src/thelab_langchain/agent/__init__.py create mode 100644 src/thelab_langchain/agent/graph.py create mode 100644 src/thelab_langchain/agent/state.py create mode 100644 src/thelab_langchain/agent/tools/__init__.py create mode 100644 src/thelab_langchain/agent/tools/memory.py create mode 100644 src/thelab_langchain/chat.py create mode 100644 src/thelab_langchain/cli.py create mode 100644 src/thelab_langchain/config.py create mode 100644 src/thelab_langchain/llm.py create mode 100644 src/thelab_langchain/voice/__init__.py create mode 100644 src/thelab_langchain/voice/audio.py create mode 100644 src/thelab_langchain/voice/orchestrator.py create mode 100644 src/thelab_langchain/voice/riva.py create mode 100644 tests/__init__.py create mode 100644 tests/test_agent_graph.py create mode 100644 tests/test_chat.py create mode 100644 tests/test_config.py create mode 100644 tests/test_llm.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a69d2aa --- /dev/null +++ b/.env.example @@ -0,0 +1,33 @@ +# Supermemory API key (get from https://console.supermemory.ai) +SUPERMEMORY_API_KEY=sm_... + +# xAI Grok API key (get from https://console.x.ai/) +XAI_API_KEY=xai_... + +# Optional: Anthropic (if you prefer Claude instead of Grok) +# ANTHROPIC_API_KEY=sk-ant-... + +# LLM configuration +LLM_PROVIDER=xai # xai | anthropic +LLM_MODEL=grok-3 # For xai: grok-3, grok-4, grok-3-latest, etc. + # For anthropic: claude-3-7-sonnet-20250219, claude-3-5-haiku-20241022, etc. +LLM_TEMPERATURE=0.7 +LLM_MAX_TOKENS=2048 + +# Default user/container for demo (change per user/session) +DEFAULT_USER_ID=demo-user + +# --- DGX Spark Desktop Local LLM (single-node v1: 120b Nemotron via NIM on :8000) --- +# After `docker compose up nemotron` (or the manual nemotron-120b container), use this. +# The 120b-class model gives the best responsive quality currently practical on one DGX Spark. +# (340b-class requires 2-3x DGX Spark nodes clustered; same agent/voice code works unchanged.) +#LLM_PROVIDER=openai_compatible +#LLM_BASE_URL=http://localhost:8000/v1 +#LLM_MODEL=nvidia/nemotron-3-super-120b-a12b +#OPENAI_API_KEY=dummy +#DEFAULT_USER_ID=grok-dgx-voice-agent +# +# Once the model is fully loaded, test with: +# make env +# make chat +# (or source .env and run the venv thelab-chat directly) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..71226d7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install minimal CPU-only dependencies + # The full package depends on audio/GPU libraries (sounddevice / PortAudio, + # nvidia-riva-client) that the voice layer needs but the tests do not. + # We install the package without its deps, then add only the lightweight + # set required to import and test the agent brain on a plain CPU runner. + run: | + python -m pip install --upgrade pip + pip install -e . --no-deps + pip install \ + "langchain-core>=0.3" \ + "langgraph>=0.2" \ + "pydantic>=2.0" \ + "pydantic-settings>=2.0" \ + "python-dotenv>=1.0" \ + "rich>=13.0" \ + "supermemory>=0.1" \ + "pytest>=8.0" \ + "pytest-asyncio>=0.23" \ + "ruff>=0.4" + + - name: Lint + run: ruff check . + + - name: Test + run: pytest -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..839618c --- /dev/null +++ b/.gitignore @@ -0,0 +1,165 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# poetry +poetry.lock + +# pdm +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# IDEs +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Project specific +supermemory.db +*.log + +# Benchmark reports - keep the structure and .gitkeep + summary files, +# but large raw logs / binaries can be ignored if they get big. +benchmarks/reports/*/ +!benchmarks/reports/*/.gitkeep +!benchmarks/reports/*/summary.md +!benchmarks/reports/*/metrics.json +!benchmarks/reports/*/events.jsonl +!benchmarks/reports/*/config.json + +# Temporary benchmark artifacts +*.bench +bench-*.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dca6bc2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# Dockerfile for the thelab-voice-agent service +# Target: DGX Spark (and general NVIDIA GPU environments) + +FROM python:3.12-slim AS builder + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Copy everything needed for metadata + build +COPY pyproject.toml . +COPY README.md . +COPY src/ ./src/ + +# Build the wheel (non-editable, clean) +RUN pip install --no-cache-dir build && \ + python -m build --wheel --outdir /wheels . + +# ---- Runtime image ---- +FROM python:3.12-slim + +# Create non-root user +RUN useradd --create-home --shell /bin/bash appuser + +WORKDIR /app + +# Audio runtime for sounddevice (used by the voice layer for mic/speakers on desktop DGX Spark). +# This is only required when running `thelab-chat voice`. The text `chat` command works without it. +RUN apt-get update && apt-get install -y --no-install-recommends \ + libportaudio2 \ + && rm -rf /var/lib/apt/lists/* + +# Install the wheel + runtime deps only +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir /wheels/*.whl && \ + chown -R appuser:appuser /app + +USER appuser + +# Default command (override in compose or at runtime) +# Example for voice: thelab-chat voice --user derek +ENTRYPOINT ["thelab-chat"] +CMD ["--help"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e6b76a0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Derek Clair + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..98fa34d --- /dev/null +++ b/Makefile @@ -0,0 +1,184 @@ +# thelab-langchain — Makefile +# All development happens inside a project-local venv to avoid dependency hell. +# Never use global pip / homebrew python for this project. +# +# For the voice spike / Lenovo Go prototype: +# Just run `make` (bare) in this directory. +# It preps the venv + package so the sibling conversational-voice-agent spike can use the real agent. +# +# Then (in the other shell): cd ../conversational-voice-agent && make +# (bare `make` there actually starts the voice loop that drives the spike.) +# +# Use `make help` for the full list of targets (chat, local, install, pids, kill, etc.). + +SHELL := /bin/bash +.DEFAULT_GOAL := local + +VENV := .venv +PYTHON := $(VENV)/bin/python +PIP := $(VENV)/bin/pip +CLI := $(VENV)/bin/thelab-chat + +.PHONY: help venv install run chat profile env lint clean local pids stop kill reset + +help: + @echo "thelab-langchain — venv-first development" + @echo "" + @echo "For the Lenovo Go voice spike (the 'two shells' flow):" + @echo " Bare 'make' (or 'make local') → prep this side so the sibling conversational-voice-agent can drive the *real* agent + Supermemory" + @echo " Then (other shell): cd ../conversational-voice-agent && make keys && make" + @echo "" + @echo "Normal development targets:" + @echo " make install Create .venv + install the package + dev tools (recommended first step)" + @echo " make run / chat Interactive chat with the agent (uses venv directly)" + @echo " make profile Show Supermemory profile for the default user" + @echo " make env Print current resolved config (keys redacted)" + @echo " make lint Run ruff + mypy" + @echo " make clean Remove venv + caches" + @echo "" + @echo "Cleanup (cross-repo aware):" + @echo " make pids List thelab + spike processes + pipes" + @echo " make stop Graceful SIGTERM" + @echo " make kill Force kill (thelab + local_tts bits)" + @echo " make reset stop + pipe cleanup (spike artifacts)" + @echo "" + @echo "See the 'local' target output for the exact next steps and key requirements." + +# Create or refresh the project venv and install the package + dev tools. +# This is the only supported way to work on this project. +venv: $(VENV)/bin/activate + +$(VENV)/bin/activate: pyproject.toml + @echo "==> Creating isolated venv at $(VENV) (no global pollution)" + python3 -m venv $(VENV) + $(PIP) install --upgrade pip wheel + $(PIP) install -e ".[dev]" + @echo "" + @echo "==> Venv ready. To activate in your shell:" + @echo " source $(VENV)/bin/activate" + @echo " (or just keep using 'make run' / 'make chat' — they use the venv directly)" + @touch $(VENV)/bin/activate + +install: venv + @echo "==> Install complete. Use 'make run' or 'make chat' to start." + +# Prep this side for the conversational-voice-agent Lenovo Go voice spike (the "two shells" flow). +# - Ensures this project's venv + editable package exist (so `pip install -e ../thelab` +# from the sibling conversational-voice-agent/.venv will succeed cleanly and pull the right code/deps). +# - The actual cross wiring + key symlink + running the voice waiter happens in the +# *conversational-voice-agent* dir via `cd conversational-voice-agent && make local` (or make demo). +# - This target is intentionally lightweight: just make sure the thelab package is +# installable from the sibling spike shell. +local: install + @echo "==> thelab prepped for the voice spike (you just ran 'make' here)." + @echo "" + @echo " Next step (other shell / other repo):" + @echo " cd ../conversational-voice-agent" + @echo " cp .env.example .env # add your keys here" + @echo " make keys # imports XAI_API_KEY + SUPERMEMORY_API_KEY into thelab/.env" + @echo " make # (or make demo for the quickest real-agent + speak test)" + @echo "" + @echo " The conversational-voice-agent Makefile will symlink thelab/.env if present so load_dotenv() works." + @echo " Once the voice loop waiter is running:" + @echo " echo 'start' > /tmp/voice_trigger" + @echo " Or run the physical Teams button listener in a third shell." + @echo "" + @echo " Watch the conversational-voice-agent terminal for real [Parakeet] Partial lines + real [AGENT] success." + @echo "" + +# All the useful targets depend on the venv existing. +run: $(CLI) + $(CLI) chat + +chat: $(CLI) + $(CLI) chat + +profile: $(CLI) + $(CLI) profile + +env: $(CLI) + $(CLI) env + +lint: $(PYTHON) + $(PYTHON) -m ruff check . + $(PYTHON) -m mypy src + +clean: + @echo "==> Removing venv and Python caches" + find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + find . -type f -name "*.pyc" -delete 2>/dev/null || true + rm -rf $(VENV) + @echo "==> Clean. Run 'make install' to recreate." + +# --- Docker / DigitalOcean Registry targets --- + +REGISTRY ?= registry.digitalocean.com/basementlab +IMAGE ?= $(REGISTRY)/thelab-agent +TAG ?= latest + +docker-build: + @echo "==> Building $(IMAGE):$(TAG)" + docker build -t $(IMAGE):$(TAG) . + +docker-push: docker-build + @echo "==> Pushing $(IMAGE):$(TAG) to DigitalOcean Container Registry" + docker push $(IMAGE):$(TAG) + +docker-run-dgx: + @echo "==> Running agent on DGX using image from registry" + REGISTRY=$(REGISTRY) TAG=$(TAG) docker compose up agent + +docker-login: + doctl registry login + +.PHONY: docker-build docker-push docker-run-dgx docker-login + +# --- Benchmark harness (Feature 007) --- + +BENCH := $(VENV)/bin/python -m benchmarks.runner + +.PHONY: benchmark benchmark-short benchmark-dry + +benchmark: $(PYTHON) + @echo "==> Running benchmark (short scenario by default). Use BENCHMARK_REPORT_DIR or the runner directly for full control." + $(BENCH) run --scenario short + +benchmark-short: benchmark + +benchmark-dry: $(PYTHON) + $(BENCH) run --scenario short --dry-run + +benchmark-help: $(PYTHON) + $(BENCH) --help + +# --- Process cleanup helpers (rogue PIDs from chat, voice, spike cross-talk, etc.) --- +# These are safe no-ops if nothing matches. Useful when you have multiple shells +# running make chat / voice_loop bits / button listeners across the two repos. + +pids status: + @echo "==> thelab-related processes:" + @pgrep -af 'thelab-chat|thelab_langchain|benchmarks\.runner' | grep -v 'pgrep -af' || echo " (none)" + @pgrep -af 'local_tts\.(voice_loop|button_listener)' | grep -v 'pgrep -af' || echo " (no cross-repo conversational-voice-agent spike processes)" + @echo "" + @echo "==> Local voice pipes (if this shell has been used for spike triggers):" + @ls -l /tmp/voice_trigger /tmp/voice_speak 2>/dev/null || echo " (no voice pipes)" + +stop: + @echo "==> Stopping thelab / spike processes (SIGTERM)..." + @pkill -f 'thelab-chat' 2>/dev/null || true + @pkill -f 'thelab_langchain.*(cli|voice)' 2>/dev/null || true + @pkill -f 'local_tts\.(voice_loop|button_listener)' 2>/dev/null || true + @echo "==> SIGTERM sent. Check with 'make pids'." + +kill: + @echo "==> Force-killing rogue thelab/spike PIDs..." + @pkill -9 -f 'thelab-chat' 2>/dev/null || true + @pkill -9 -f 'thelab_langchain' 2>/dev/null || true + @pkill -9 -f 'local_tts\.(voice_loop|button_listener)' 2>/dev/null || true + @echo "==> Force kill done." + +reset: stop + @echo "==> Also cleaning voice pipes (spike artifacts)..." + @rm -f /tmp/voice_trigger /tmp/voice_speak 2>/dev/null || true + @echo "==> Reset complete. cd ../conversational-voice-agent && make reset (or make audio-reset) if you also need audio cleanup." + @echo " Then 'make local' (or bare 'make') in the appropriate repo to restart." diff --git a/README.md b/README.md index ea3ccd9..24fa51a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,213 @@ -# thelab -Memory-aware LangGraph agent brain (Supermemory + pluggable Grok/Claude/local NIM) — companion to conversational-voice-agent +# thelab-langchain + +A small, memory-aware **LangGraph agent brain**. This is a personal learning project that pairs a LangGraph reasoning graph with [Supermemory](https://supermemory.ai) for long-term recall, and can talk to **Grok (xAI)**, **Claude (Anthropic)**, or any OpenAI-compatible local model (e.g. a Nemotron NIM on a DGX Spark). It is meant to be a clean, readable reference for wiring memory into an agent — not a production framework, so expect rough edges. + +What it demonstrates: + +- Long-term user memory and automatic profiling via [Supermemory](https://supermemory.ai) +- A LangGraph graph that proactively injects relevant memory, then lets the model call memory tools when it wants them +- Pluggable LLM providers behind a single factory (Grok by default — no OpenAI dependency required) +- A simple interactive CLI + +> **Used by:** the companion [`conversational-voice-agent`](https://github.com/derekclair/conversational-voice-agent) repo — a local voice front-end (STT/TTS on a DGX Spark) that uses this package as its agent brain. + +## Prerequisites + +- Python ≥ 3.11 +- API keys for: + - **Supermemory**: https://console.supermemory.ai + - **Grok (xAI)**: https://console.x.ai/ (recommended) + - or **Anthropic**: https://console.anthropic.com + +## Quick Start (venv enforced) + +**We use a project-local `.venv` only.** This prevents dependency collisions with anything else on your machine (especially important with Homebrew Python, other projects, etc.). + +1. (Optional but recommended) Copy the example env and fill your keys: + + ```bash + cp .env.example .env + # Then edit .env with your SUPERMEMORY_API_KEY and XAI_API_KEY (or ANTHROPIC) + ``` + +2. One command to rule them all: + + ```bash + make install + ``` + + This will: + - Create a fresh `.venv/` in the project root + - Install the package + dev tools in complete isolation + - Never touch your global Python environment + +3. Run the chat (still using the venv automatically): + + ```bash + make chat + # or + make run + ``` + + Or with a named user/container (Supermemory isolates by `container_tag`): + + ```bash + make run # then inside the chat: /user alice + ``` + + You can also activate the venv the normal way if you prefer: + + ```bash + source .venv/bin/activate + thelab-chat chat --user derek + ``` + +**Important**: `.env` is gitignored and should **never** be committed. Your new dedicated Supermemory key will stay local. + + ```bash + thelab-chat chat --user alice + ``` + +### Special Commands (inside the chat) + +| Command | Effect | +|---------------|--------| +| `/profile` | Show current Supermemory profile + facts | +| `/clear` | Reset local conversation buffer (long-term memory stays) | +| `/user ` | Switch to a different user/container | +| `/quit` | Exit | +| `/help` | List commands | + +Everything else is sent to the LLM together with rich memory context pulled from Supermemory. + +## How It Works + +On every turn the agent: + +1. Calls `memory.profile(container_tag=user_id, q=message)` — Supermemory returns: + - `static` facts (long-term profile) + - `dynamic` context (recent activity) + - Semantically relevant past memories + +2. Injects a nicely formatted context block into the system prompt. + +3. Calls the chosen LLM (`ChatXAI` or `ChatAnthropic`). + +4. Stores the turn via `memory.add(...)` so future conversations remember it. + +This pattern gives you excellent personalization and continuity without managing your own vector store or prompt engineering for memory. + +## About the Official Supermemory + LangChain Guide + +The official docs at https://supermemory.ai/docs/integrations/langchain currently show this as the "next step": + +```python +from langchain_openai import ChatOpenAI +from supermemory import Supermemory + +memory = Supermemory() +llm = ChatOpenAI(model="gpt-4o") +... +``` + +**This is just an example**, not a requirement. + +Supermemory is a standalone memory service. The `Supermemory()` client is completely decoupled from which LLM provider you use. You can (and we do) pair it with `ChatXAI`, `ChatAnthropic`, or any other LangChain chat model. + +See [examples/official_guide_style.py](examples/official_guide_style.py) for a drop-in version of the exact snippet from the guide, but using Grok instead of OpenAI. + +## Switching to Anthropic / Claude + +In `.env`: + +```env +LLM_PROVIDER=anthropic +LLM_MODEL=claude-3-7-sonnet-20250219 +ANTHROPIC_API_KEY=sk-ant-... +``` + +Then install the optional extra (the import is lazy) — best done via the venv: + +```bash +make install # already includes dev tools +# or after venv exists: +.venv/bin/pip install -e ".[anthropic]" +``` + +## Project Layout + +``` +. +├── src/thelab_langchain/ +│ ├── __init__.py +│ ├── config.py # Pydantic settings + validation +│ ├── llm.py # LLM provider factory (Grok / Anthropic / OpenAI-compatible) +│ ├── chat.py # MemoryChat core (profile → LLM → store) +│ ├── cli.py # Typer + Rich interactive shell +│ ├── agent/ # LangGraph agent brain +│ │ ├── graph.py # Reasoning graph (memory injection + tool calls) +│ │ ├── state.py # Graph state definitions +│ │ └── tools/ # Agent tools (e.g. Supermemory memory tool) +│ └── voice/ # Optional voice front-end (STT/TTS orchestration) +│ ├── orchestrator.py +│ ├── audio.py +│ └── riva.py +├── examples/ # Runnable, non-interactive usage examples +├── .env.example +├── pyproject.toml +└── README.md +``` + +## Documentation & Getting Started + +- **[Development Guide](docs/development.md)** — How to set up your environment, run text vs voice mode, and common commands. +- **[Architecture Overview](docs/architecture.md)** — Layering, key components, and how everything fits together. +- `specs/` — Feature specifications and design decisions (read these to understand *why* things are built the way they are). + +## Development + +### Local (Mac) Development + +All commands go through the local venv via Make: + +```bash +make install # first time / after clean +make lint # ruff + mypy +make chat # text chat demo +make run # same as chat +``` + +### Docker + DGX Spark Deployment (Recommended for Voice) + +The canonical way to run the full stack (agent + Riva + Nemotron) is via Docker Compose on the DGX Spark: + +```bash +# 1. Develop on Mac +# 2. Build the agent image +docker compose build + +# 3. On the DGX (ssh dgx) +docker compose pull # or build +docker compose up -d +``` + +See `docker-compose.yml` for the current services (`agent`, `riva`, `nemotron`). + +The agent can be pointed at either Grok or the local Nemotron NIM at runtime via environment variables. + +(Full observability is deferred for now.) + +## Next Steps / Ideas + +- Add proper LangGraph agent with tools + Supermemory as a tool +- Persistent local conversation history + summarization +- Metadata filtering examples (`memory.search.memories(filters=...)`) +- Evaluation harness against MemoryBench +- Expose as a FastAPI service or Discord/Slack bot +- Store documents (not just chat turns) via `memory.add(url=...)` or raw content + +## References + +- Supermemory LangChain guide: https://supermemory.ai/docs/integrations/langchain +- langchain-xai docs: https://python.langchain.com/docs/integrations/chat/xai +- xAI API: https://docs.x.ai/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..b0d7535 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,70 @@ +# Benchmark Harness for DGX Spark Voice Agent + +This directory contains the measurement tooling for **Feature 007: DGX Hardware Optimization & Sweet-Spot Discovery**. + +## Purpose + +We need hard numbers, not vibes. + +Every optimization (audio stack changes, model swaps, context tuning, etc.) must be validated against a reproducible baseline captured on the real DGX Spark hardware. + +## Quick Start (on DGX) + +```bash +# Make sure the agent + Riva/NIM services are up +docker compose up -d + +# Run a short-turn benchmark session (records metrics + timing) +python -m benchmarks.runner --scenario short --user derek --output-dir benchmarks/reports/my-run + +# Or after we add the entry point: +# thelab-bench --scenario short ... +``` + +## Directory Layout + +``` +benchmarks/ +├── __init__.py +├── README.md +├── runner.py # Main CLI / harness +├── reports/ # Timestamped result directories (committed) +│ └── 2025-05-22-baseline-120b-riva/ +│ ├── summary.md +│ ├── metrics.json +│ └── raw/ +└── (future) timing.py, metrics.py, comparators.py +``` + +## Reports + +Each run creates a directory with: +- `summary.md` — human readable key metrics + notes +- `metrics.jsonl` or `metrics.json` — structured data for comparison scripts +- `events.jsonl` — detailed timestamped events from the orchestrator +- `nvidia-smi.log`, `docker-stats.log`, etc. +- `config.json` — exact image tags, env vars, compose profile used + +## Instrumentation + +The voice loop is lightly instrumented when `BENCHMARK_MODE=1` is set. + +See `src/thelab_langchain/voice/orchestrator.py` for the event hooks. + +## Scenarios + +- `short`: Quick Q&A + memory recall turns (default for latency) +- `long`: Extended household conversation (tests context / KV pressure) +- `concurrent`: Multiple simulated users (future) + +## Comparison + +Later we will add `python -m benchmarks.compare report1 report2` to generate delta tables. + +## Related + +- [specs/007-dgx-hardware-optimization/spec.md](../specs/007-dgx-hardware-optimization/spec.md) +- [specs/007-dgx-hardware-optimization/plan.md](../specs/007-dgx-hardware-optimization/plan.md) +- [specs/007-dgx-hardware-optimization/tasks.md](../specs/007-dgx-hardware-optimization/tasks.md) + +Let's measure what actually happens on the hardware. No guessing. \ No newline at end of file diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e860165 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1,16 @@ +""" +Benchmark harness for the thelab-langchain voice agent on DGX Spark. + +This package provides tools to measure: +- Voice turn latency (end-of-speech → first audio) +- LLM TTFT + generation speed +- Memory / VRAM usage under load +- Concurrency behavior +- etc. + +All reports go under benchmarks/reports/ and are committed with pinned configs. + +See specs/007-dgx-hardware-optimization/ for the overall plan and tasks. +""" + +__version__ = "0.1.0" \ No newline at end of file diff --git a/benchmarks/metrics.py b/benchmarks/metrics.py new file mode 100644 index 0000000..7877d1e --- /dev/null +++ b/benchmarks/metrics.py @@ -0,0 +1,270 @@ +""" +System and GPU metrics sampler for benchmark runs. + +Collects time-series data during voice agent runs so we can measure: +- Peak / average GPU memory usage (critical on 128 GB unified DGX Spark) +- Container memory (Riva, Nemotron, agent) +- CPU / power / thermals where available + +Designed to be lightweight and safe to run alongside the voice loop. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import threading +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Optional + + +def _now() -> float: + return time.time() + + +def _iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +class MetricsSampler: + """ + Background sampler that periodically collects system/GPU metrics + and writes them as JSON lines into a report directory. + """ + + def __init__( + self, + report_dir: Path | str, + interval: float = 2.0, + containers: Optional[list[str]] = None, + enabled: bool = True, + ): + self.report_dir = Path(report_dir) + self.interval = interval + self.containers = containers or ["riva", "nemotron", "agent", "thelab"] + self.enabled = enabled and self._has_required_tools() + + self._stop_event = threading.Event() + self._thread: Optional[threading.Thread] = None + + self.gpu_file = self.report_dir / "gpu_samples.jsonl" + self.docker_file = self.report_dir / "docker_stats.jsonl" + self.system_file = self.report_dir / "system_samples.jsonl" + + self.report_dir.mkdir(parents=True, exist_ok=True) + + def _has_required_tools(self) -> bool: + """Check if we can actually collect useful data.""" + has_nvidia = shutil.which("nvidia-smi") is not None + has_docker = shutil.which("docker") is not None + return has_nvidia or has_docker + + def _sample_gpu(self) -> dict[str, Any]: + """Sample NVIDIA GPU using nvidia-smi (works great on DGX Spark).""" + if not shutil.which("nvidia-smi"): + return {"available": False} + + try: + # Query key fields in CSV for easy parsing + cmd = [ + "nvidia-smi", + "--query-gpu=index,name,memory.total,memory.used,memory.free,utilization.gpu,utilization.memory,power.draw,temperature.gpu", + "--format=csv,noheader,nounits", + ] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=5) + if result.returncode != 0: + return {"error": result.stderr.strip()[:200]} + + lines = [line.strip() for line in result.stdout.strip().split("\n") if line.strip()] + gpus = [] + for line in lines: + parts = [p.strip() for p in line.split(",")] + if len(parts) >= 9: + gpus.append({ + "index": int(parts[0]), + "name": parts[1], + "mem_total_mb": float(parts[2]), + "mem_used_mb": float(parts[3]), + "mem_free_mb": float(parts[4]), + "gpu_util_pct": float(parts[5]), + "mem_util_pct": float(parts[6]), + "power_w": float(parts[7]) if parts[7] else None, + "temp_c": float(parts[8]) if parts[8] else None, + }) + return {"available": True, "gpus": gpus} + except Exception as e: + return {"available": False, "error": str(e)} + + def _sample_docker(self) -> dict[str, Any]: + """Sample memory/CPU for relevant containers.""" + if not shutil.which("docker"): + return {"available": False} + + samples: dict[str, Any] = {} + for name in self.containers: + try: + # Use docker stats once (non-streaming) + cmd = [ + "docker", "stats", name, + "--no-stream", + "--format", "{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}}", + ] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=4) + if result.returncode == 0 and result.stdout.strip(): + line = result.stdout.strip().split("\n")[0] + parts = [p.strip() for p in line.split(",")] + if len(parts) >= 3: + samples[name] = { + "cpu_pct": parts[1], + "mem_usage": parts[2], + "mem_pct": parts[3] if len(parts) > 3 else None, + } + except Exception: + continue + + return {"available": bool(samples), "containers": samples} + + def _sample_system(self) -> dict[str, Any]: + """Basic system memory (works on macOS and Linux/DGX).""" + try: + # Use vm_stat on mac, /proc/meminfo on Linux + if os.path.exists("/proc/meminfo"): + with open("/proc/meminfo") as f: + meminfo = f.read() + # Very rough parse for MemTotal / MemAvailable + total = None + available = None + for line in meminfo.splitlines(): + if line.startswith("MemTotal:"): + total = int(line.split()[1]) # kB + if "MemAvailable:" in line: + available = int(line.split()[1]) + if total and available: + used = total - available + return { + "total_kb": total, + "used_kb": used, + "available_kb": available, + "used_pct": round(used / total * 100, 1), + } + # Fallback: use psutil if available (not a hard dep) + try: + import psutil # type: ignore + vm = psutil.virtual_memory() + return { + "total_bytes": vm.total, + "used_bytes": vm.used, + "available_bytes": vm.available, + "used_pct": vm.percent, + } + except ImportError: + pass + except Exception: + pass + return {"available": False} + + def _write_sample(self, filename: Path, data: dict[str, Any]) -> None: + data = {"ts": _now(), "iso": _iso(), **data} + with filename.open("a") as f: + f.write(json.dumps(data, default=str) + "\n") + + def _sample_loop(self) -> None: + """Main sampling loop.""" + while not self._stop_event.is_set(): + try: + gpu = self._sample_gpu() + if gpu.get("available"): + self._write_sample(self.gpu_file, {"type": "gpu", **gpu}) + + docker = self._sample_docker() + if docker.get("available"): + self._write_sample(self.docker_file, {"type": "docker", **docker}) + + system = self._sample_system() + if system.get("used_pct") is not None or system.get("available"): + self._write_sample(self.system_file, {"type": "system", **system}) + + except Exception as e: + # Never let the sampler crash the benchmark + self._write_sample( + self.report_dir / "sampler_errors.jsonl", + {"type": "sampler_error", "error": str(e)}, + ) + + self._stop_event.wait(self.interval) + + def start(self) -> None: + """Start background sampling (no-op if disabled or already running).""" + if not self.enabled or self._thread and self._thread.is_alive(): + return + + self._stop_event.clear() + self._thread = threading.Thread(target=self._sample_loop, daemon=True, name="metrics-sampler") + self._thread.start() + + def stop(self) -> None: + """Stop the sampler cleanly.""" + if self._stop_event: + self._stop_event.set() + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=5) + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + + +def quick_peak_analysis(report_dir: Path | str) -> dict[str, Any]: + """Very lightweight post-run analysis to extract peaks from collected samples.""" + report_dir = Path(report_dir) + peaks: dict[str, Any] = {} + + # GPU peak memory + gpu_file = report_dir / "gpu_samples.jsonl" + if gpu_file.exists(): + max_used = 0.0 + for line in gpu_file.read_text().splitlines(): + try: + data = json.loads(line) + for g in data.get("gpus", []): + max_used = max(max_used, g.get("mem_used_mb", 0)) + except Exception: + continue + if max_used > 0: + peaks["gpu_mem_peak_mb"] = round(max_used, 1) + + # System memory peak (rough) + sys_file = report_dir / "system_samples.jsonl" + if sys_file.exists(): + max_used_pct = 0.0 + for line in sys_file.read_text().splitlines(): + try: + data = json.loads(line) + pct = data.get("used_pct", 0) + if isinstance(pct, (int, float)): + max_used_pct = max(max_used_pct, pct) + except Exception: + continue + if max_used_pct > 0: + peaks["system_mem_peak_pct"] = round(max_used_pct, 1) + + return peaks + + +if __name__ == "__main__": + # Quick manual test + import tempfile + with tempfile.TemporaryDirectory() as tmp: + sampler = MetricsSampler(tmp, interval=1.0) + print("Sampler enabled:", sampler.enabled) + sampler.start() + time.sleep(3) + sampler.stop() + print("Sample files created:", list(Path(tmp).glob("*.jsonl"))) \ No newline at end of file diff --git a/benchmarks/reports/.gitkeep b/benchmarks/reports/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/runner.py b/benchmarks/runner.py new file mode 100644 index 0000000..f0df27d --- /dev/null +++ b/benchmarks/runner.py @@ -0,0 +1,218 @@ +""" +Benchmark runner for the thelab voice agent on DGX Spark. + +This is the entry point for controlled measurement runs. + +Usage (early skeleton): + python -m benchmarks.runner --scenario short --user derek --output-dir benchmarks/reports/test-run + +Later this will become `thelab-bench` and grow rich timing + metrics collection. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Annotated, Any + +import typer +from rich.console import Console +from rich.panel import Panel + +from .metrics import MetricsSampler, quick_peak_analysis + +console = Console() +app = typer.Typer( + name="thelab-bench", + help="Benchmark harness for thelab-langchain voice agent (DGX Spark optimization)", + add_completion=False, +) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _create_report_dir(base: Path, scenario: str) -> Path: + """Create a timestamped report directory.""" + ts = datetime.now().strftime("%Y-%m-%d_%H%M%S") + report_dir = base / f"{ts}_{scenario}" + report_dir.mkdir(parents=True, exist_ok=True) + return report_dir + + +def _write_config_snapshot(report_dir: Path, extra: dict[str, Any] | None = None) -> Path: + """Capture environment and settings for reproducibility.""" + cfg = { + "timestamp": _now_iso(), + "scenario": extra.get("scenario") if extra else None, + "user": extra.get("user") if extra else None, + "env": { + k: v + for k, v in os.environ.items() + if any(x in k.upper() for x in ("RIVA", "NIM", "LLM", "BENCHMARK", "USER")) + or k in ("DOCKER_HOST",) + }, + "python": sys.version, + "cwd": str(Path.cwd()), + } + if extra: + cfg.update(extra) + + path = report_dir / "config.json" + path.write_text(json.dumps(cfg, indent=2, default=str)) + return path + + +def _log_event(report_dir: Path, event: dict[str, Any]) -> None: + """Append a structured event line.""" + events_file = report_dir / "events.jsonl" + event = {"ts": _now_iso(), **event} + with events_file.open("a") as f: + f.write(json.dumps(event) + "\n") + + +@app.command() +def run( + scenario: Annotated[ + str, + typer.Option("--scenario", "-s", help="Benchmark scenario: short | long | concurrent"), + ] = "short", + user: Annotated[ + str | None, + typer.Option("--user", "-u", help="User ID (Supermemory container)"), + ] = None, + output_dir: Annotated[ + Path, + typer.Option("--output-dir", "-o", help="Base directory for reports"), + ] = Path("benchmarks/reports"), + duration: Annotated[ + int | None, + typer.Option("--duration", "-d", help="Max seconds to run (None = until Ctrl+C)"), + ] = None, + dry_run: Annotated[ + bool, + typer.Option("--dry-run", help="Just set up the report dir and print what would happen"), + ] = False, +) -> None: + """ + Run a benchmark session and capture timing + system metrics. + + This is the primary entry point for Phase 0 baseline capture and all subsequent experiments. + """ + user_id = user or os.getenv("DEFAULT_USER_ID", "grok-dgx-voice-agent") + + console.print( + Panel.fit( + f"[bold cyan]thelab-bench[/bold cyan]\n\n" + f"Scenario : [green]{scenario}[/green]\n" + f"User : [cyan]{user_id}[/cyan]\n" + f"Output : [magenta]{output_dir}[/magenta]\n" + f"Duration : {duration or 'until Ctrl+C'}s\n\n" + "Setting BENCHMARK_MODE=1 for instrumentation.", + title="Benchmark Run Starting", + border_style="cyan", + ) + ) + + report_dir = _create_report_dir(output_dir, scenario) + console.print(f"[green]Report directory:[/green] {report_dir}") + + # Capture starting state + _write_config_snapshot(report_dir, {"scenario": scenario, "user": user_id}) + _log_event(report_dir, {"type": "benchmark_start", "scenario": scenario, "user_id": user_id}) + + if dry_run: + console.print("[yellow]Dry run complete. No voice session launched.[/yellow]") + _log_event(report_dir, {"type": "dry_run_complete"}) + return + + # Set benchmark mode so the orchestrator emits timing events + env = os.environ.copy() + env["BENCHMARK_MODE"] = "1" + env["BENCHMARK_REPORT_DIR"] = str(report_dir) + if user_id: + env["DEFAULT_USER_ID"] = user_id + + # Start background system/GPU metrics sampler (very valuable on DGX Spark) + sampler = MetricsSampler( + report_dir=report_dir, + interval=2.0, + containers=["riva", "nemotron", "agent", "thelab-agent"], + ) + sampler.start() + _log_event(report_dir, {"type": "metrics_sampler_started", "enabled": sampler.enabled}) + + # For now, we invoke the existing thelab-chat voice command. + # In a more advanced version we will call the orchestrator directly with hooks. + cmd = [sys.executable, "-m", "thelab_langchain.cli", "voice", "--user", user_id] + + console.print(f"\n[bold]Launching voice session with BENCHMARK_MODE=1...[/bold]") + console.print(f"[dim]Command:[/dim] {' '.join(cmd)}") + console.print("[yellow]Speak normally. Press Ctrl+C when finished with the benchmark run.[/yellow]\n") + + start_time = time.time() + + try: + proc = subprocess.Popen(cmd, env=env) + if duration: + try: + proc.wait(timeout=duration) + except subprocess.TimeoutExpired: + proc.terminate() + proc.wait() + else: + proc.wait() + except KeyboardInterrupt: + console.print("\n[yellow]Benchmark run interrupted by user.[/yellow]") + if "proc" in locals(): + proc.terminate() + proc.wait() + finally: + # Stop the metrics sampler first + sampler.stop() + _log_event(report_dir, {"type": "metrics_sampler_stopped"}) + + elapsed = time.time() - start_time + _log_event( + report_dir, + { + "type": "benchmark_end", + "elapsed_seconds": round(elapsed, 2), + "returncode": getattr(proc, "returncode", None) if "proc" in locals() else None, + }, + ) + + # Quick automatic peak analysis (extremely useful for headroom decisions) + peaks = quick_peak_analysis(report_dir) + if peaks: + _log_event(report_dir, {"type": "peak_analysis", **peaks}) + + # Write a minimal summary + summary = report_dir / "summary.md" + summary.write_text( + f"# Benchmark Run Summary\n\n" + f"**Scenario**: {scenario}\n" + f"**User**: {user_id}\n" + f"**Started**: {_now_iso()}\n" + f"**Duration**: {elapsed:.1f}s\n\n" + f"**Peaks (auto-detected)**:\n" + f"{json.dumps(peaks, indent=2) if peaks else ' (no peaks extracted - check nvidia-smi/docker availability)'}\n\n" + f"See `events.jsonl`, `gpu_samples.jsonl`, `docker_stats.jsonl`, and `config.json` for full details.\n\n" + f"---\n\n" + f"Next step: run comparison tooling against baseline.\n" + ) + + console.print(f"\n[green]Benchmark complete.[/green] Report written to: {report_dir}") + console.print(f"[dim]Key files: events.jsonl, gpu_samples.jsonl, docker_stats.jsonl, summary.md[/dim]") + if peaks: + console.print(f"[cyan]Auto-detected peaks:[/cyan] {peaks}") + + +if __name__ == "__main__": + app() \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5b11b65 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,132 @@ +# docker-compose.yml +# Canonical way to run the TheLab Voice Agent stack on DGX Spark. +# +# For single-node DGX Spark desktop (current v1): uses 120b Nemotron NIM by default (responsive voice latency). +# For future multi-node 340b-class: set NEMOTRON_IMAGE and NEMOTRON_MODEL env overrides before `docker compose up`. +# +# Development flow: +# 1. Develop on Mac +# 2. docker compose build (or buildx for multi-arch) +# 3. Push image to a registry (or build directly on DGX) +# 4. On DGX: docker compose pull && docker compose up +# +# Services: +# - agent: Our Python LangGraph + voice orchestrator (points at nemotron:8000 by default) +# - riva: NVIDIA Riva (ASR + TTS) - note: current 2.15.0 image does not support GB10; voice may need host passthrough or updated image +# - nemotron: Local Nemotron LLM via official NIM (OpenAI-compatible /v1) - defaults to 120b for desktop + +services: + agent: + build: + context: . + dockerfile: Dockerfile + image: ${REGISTRY:-registry.digitalocean.com/basementlab}/thelab-agent:${TAG:-latest} + container_name: thelab-agent + depends_on: + riva: + condition: service_healthy + nemotron: + condition: service_healthy + environment: + - PYTHONUNBUFFERED=1 + - RIVA_URI=riva:50051 + # Point the agent at the local Nemotron NIM (OpenAI compatible) + - LLM_PROVIDER=${LLM_PROVIDER:-nemotron} + - NEMOTRON_BASE_URL=http://nemotron:8000/v1 + - NEMOTRON_MODEL=${NEMOTRON_MODEL:-nvidia/nemotron-4-340b-instruct} + - SUPERMEMORY_API_KEY=${SUPERMEMORY_API_KEY} + - XAI_API_KEY=${XAI_API_KEY} + # Optional: useful for local development against host services + - HOST_IP=${HOST_IP:-host.docker.internal} + volumes: + - ./data:/app/data + # GPU access (DGX Spark) - adjust count as needed + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import thelab_langchain; print('ok')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + profiles: ["full", "agent"] + + riva: + image: nvcr.io/nvidia/riva/riva-speech:2.15.0 + container_name: riva + ports: + - "50051:50051" # gRPC (primary for our agent) + - "9000:9000" # HTTP (debug / health) + environment: + - NVIDIA_VISIBLE_DEVICES=0 + volumes: + - riva_models:/data + - riva_models:/opt/riva/models + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + shm_size: 1g + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/v1/health/ready"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 120s # Riva can take a long time to load models + profiles: ["full", "riva", "riva-only"] + + nemotron: + # NVIDIA NIM for Nemotron (OpenAI-compatible /v1 endpoint). + # Default: 120b-class model for responsive single-node DGX Spark desktop (v1 target). + # Override for future multi-DGX 340b-class: NEMOTRON_IMAGE=...-340b... NEMOTRON_MODEL=... + image: ${NEMOTRON_IMAGE:-nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b:latest} + container_name: nemotron + ports: + - "8000:8000" + environment: + - NIM_MODEL_NAME=${NEMOTRON_MODEL:-nvidia/nemotron-3-super-120b-a12b} + - NIM_SERVER_PORT=8000 + - NIM_HTTP_API_PORT=8000 + - NGC_API_KEY=${NGC_API_KEY} + volumes: + # Align with the path the NIM entrypoint actually uses for its HF/NGC cache + - nemotron_models:/opt/nim/.cache + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: unless-stopped + profiles: ["full", "nemotron", "nemotron-only"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/v1/health/ready"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 300s # 120b+ models can take 10-30+ min on first load even with cache + +volumes: + riva_models: + driver: local + nemotron_models: + driver: local + +# Usage examples: +# docker compose up # full stack (agent + riva + nemotron) +# docker compose --profile riva-only up +# docker compose --profile nemotron-only up +# +# To run only the agent against services running on the host (useful for local dev): +# HOST_IP=host.docker.internal docker compose up agent diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..db1464a --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,63 @@ +# Architecture Overview + +The system is deliberately layered so that the "brain" can evolve independently of the voice hardware and the deployment target (Mac dev vs DGX Spark personal desktop, with a clear future path to multi-node DGX Spark clusters for 340b-class models). + +## High-Level Layers + +``` +User (Voice) + │ + ▼ +Voice Layer (Riva / NeMo ASR + TTS) + │ (audio in → transcript, text out → audio) + ▼ +Orchestration Layer (VoiceOrchestrator) + │ (turn management, barge-in, audio bridging) + ▼ +Agent Brain (LangGraph) + │ (memory_injection + LLM with tools) + │ + ├── Supermemory (long-term, user-scoped) + ├── Configured LLM (Grok via XAI or Nemotron NIM locally) + └── Future tools +``` + +## Key Components + +- **`thelab_langchain.voice`** — Audio I/O + Riva client wrappers + turn management. +- **`thelab_langchain.agent`** — LangGraph graph, tools (currently Supermemory), state, and LLM factory. +- **`thelab_langchain.llm`** — Single place that returns the right chat model based on `LLM_PROVIDER`. +- **Supermemory** — Long-term memory store (profile + semantic search). Scoped per user via `container_tag`. +- **Riva** — NVIDIA's production ASR/TTS service (runs as separate container on DGX). +- **Nemotron** — Local LLM brain via NVIDIA NIM (OpenAI-compatible). Swappable at runtime with Grok. + +## Multi-User Considerations + +See `specs/002-multi-user-support/spec.md`. The architecture was designed with per-user `container_tag` (Supermemory) and per-user `thread_id` (LangGraph) from the beginning. + +## Deployment Model + +- **Development**: Mac + local `.venv` (or Docker with host networking). +- **Production / Voice**: Full Docker Compose stack on DGX Spark (`agent` + `riva` + `nemotron`). +- Images are built on the dev machine (or CI) and pushed to a private registry, then pulled on the DGX. + +See `specs/003-deployment-infrastructure/spec.md` for current gaps and target state. + +## Persistence + +- **Long-term**: Supermemory (cloud, user-scoped). +- **Short-term / Conversation**: LangGraph checkpointer (planned — see `specs/004-persistence-checkpointers/spec.md`). Currently in-memory only. + +## Extensibility + +- New tools → add in `agent/tools/`, expose via factory, wire into graph. +- New memory backends → implement the same tool interface or extend the injection node. +- New LLM providers → extend `config.py` + `llm.py` (must be OpenAI-compatible or have a LangChain integration). + +## Where to Start Exploring the Code + +1. `src/thelab_langchain/cli.py` — entry points (`chat`, `voice`, etc.) +2. `src/thelab_langchain/voice/orchestrator.py` — how voice turns become agent calls +3. `src/thelab_langchain/agent/graph.py` — the current brain (memory injection + LLM + tools) +4. `src/thelab_langchain/agent/tools/memory.py` — how we talk to Supermemory +5. `docker-compose.yml` + `Dockerfile` — how everything runs on DGX diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..461dd5b --- /dev/null +++ b/docs/development.md @@ -0,0 +1,98 @@ +# Development Guide + +## Environment Setup (Recommended) + +We strongly prefer a project-local virtual environment to avoid dependency hell (especially on macOS with Homebrew Python). + +```bash +# One-time setup +make install + +# Activate (optional — most `make` targets work without it) +source .venv/bin/activate +``` + +After `make install` you should have a working `thelab-chat` command. + +## Running the Text Agent + +```bash +# Basic text chat (uses Grok by default via .env) +make chat + +# Or directly +thelab-chat chat --user derek --thread morning-standup +``` + +## Running the Voice Agent (Local) + +The voice path requires a running Riva server (and optionally a local Nemotron). + +**Quick local test (mocked audio path coming soon):** + +```bash +# Point at services running on your host (from inside Docker or natively) +HOST_IP=host.docker.internal thelab-chat voice --user derek +``` + +For full local voice development without Docker, you will need: +- A local Riva installation or the Riva Docker container +- A local LLM (Ollama, vLLM, or NVIDIA NIM) exposing an OpenAI-compatible endpoint + +Set these environment variables: + +```env +LLM_PROVIDER=openai_compatible +LLM_BASE_URL=http://localhost:8000/v1 +RIVA_URI=localhost:50051 +``` + +## Environment Variables Reference + +See `.env.example` for the current list. Key ones: + +- `LLM_PROVIDER` — `xai` (Grok), `anthropic`, or `openai_compatible` +- `LLM_BASE_URL` — only needed for `openai_compatible` (e.g. your Nemotron NIM) +- `RIVA_URI` — address of the Riva gRPC server +- `SUPERMEMORY_API_KEY` — required +- `XAI_API_KEY` / `ANTHROPIC_API_KEY` + +## Docker Development + +See the root `docker-compose.yml` and the profiles it supports: + +```bash +# Full stack (agent + Riva + Nemotron) +docker compose up + +# Just the agent (talking to host services) +docker compose up agent +``` + +## Running on DGX Spark + +See the deployment workflow in `specs/001-voice-dgx-spark-agent/plan.md` and `specs/003-deployment-infrastructure/spec.md`. + +Typical flow: +1. Develop on Mac +2. `docker compose build` +3. Push image to your private registry +4. On the DGX: `docker compose pull && docker compose up` + +## Adding New Tools or Memory Systems + +1. Create the tool(s) in `src/thelab_langchain/agent/tools/` +2. Use the factory pattern (`create_xxx_tools(user_id)`) so they are user-scoped. +3. Wire them in `agent/graph.py` (either via proactive injection or by binding to the LLM). +4. Update the voice orchestrator if the new tools need special handling from the audio layer. + +See the existing Supermemory tools as the reference implementation. + +## Common Make Targets + +- `make install` — create venv + install +- `make chat` / `make run` — text chat +- `make lint` +- `make clean` — remove venv and caches + +Add new targets to the `Makefile` as the project grows. diff --git a/examples/basic_chat.py b/examples/basic_chat.py new file mode 100644 index 0000000..da0743c --- /dev/null +++ b/examples/basic_chat.py @@ -0,0 +1,31 @@ +"""Minimal non-interactive example of MemoryChat with Grok + Supermemory.""" + +import os +from dotenv import load_dotenv + +# Make sure we can run from anywhere +load_dotenv() + +from thelab_langchain.chat import MemoryChat # noqa: E402 + +def main() -> None: + user_id = os.getenv("DEFAULT_USER_ID", "example-user") + + print(f"Starting MemoryChat for user: {user_id}\n") + + agent = MemoryChat(user_id=user_id) + + # First turn — user introduces themselves + reply = agent.chat("Hi! My name is Alex and I really love functional programming in Python.") + print("Assistant:", reply, "\n") + + # Second turn — should recall the preference + reply = agent.chat("What kind of code style do I prefer?") + print("Assistant:", reply, "\n") + + print("--- Profile after two turns ---") + agent.show_profile() + + +if __name__ == "__main__": + main() diff --git a/examples/official_guide_style.py b/examples/official_guide_style.py new file mode 100644 index 0000000..516858a --- /dev/null +++ b/examples/official_guide_style.py @@ -0,0 +1,42 @@ +""" +Minimal example that follows the official Supermemory "next steps" guide exactly, +but using Grok (xAI) via langchain-xai instead of OpenAI. + +See: https://supermemory.ai/docs/integrations/langchain + +This proves that Supermemory is LLM-agnostic — you can (and should) use it with +whatever chat model you prefer. +""" + +import os +from dotenv import load_dotenv + +load_dotenv() + +# --- The only two changes from the official guide snippet --- +from langchain_xai import ChatXAI # instead of langchain_openai.ChatOpenAI +from supermemory import Supermemory + +memory = Supermemory() +llm = ChatXAI(model=os.getenv("LLM_MODEL", "grok-3")) +# ------------------------------------------------------------ + +# Retrieve context (exactly as shown in the guide) +result = memory.profile(container_tag="user-123", q="preferences") +context = result.profile.static or [] + +print("=== Supermemory returned context ===") +print(context or "(no static facts yet)") +print() + +# Use in chain (using the modern LangChain message format) +# NOTE: assign the joined string to a variable first — a backslash inside an +# f-string expression is a SyntaxError on Python < 3.12 (our minimum is 3.11). +context_lines = "\n".join(context) +response = llm.invoke([ + {"role": "system", "content": f"User context:\n{context_lines}"}, + {"role": "user", "content": "Help me with my project"}, +]) + +print("=== LLM response ===") +print(response.content) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..095df67 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,86 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "thelab-langchain" +version = "0.1.0" +description = "LangChain + Supermemory integration demo using Grok (xAI) or Anthropic" +readme = "README.md" +requires-python = ">=3.11" +license = {text = "MIT"} +authors = [ + {name = "Derek Clair", email = "derek@derekclair.com"} +] +keywords = ["langchain", "supermemory", "grok", "xai", "anthropic", "memory", "ai", "rag"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +dependencies = [ + "langchain>=0.3.0", + "langchain-xai>=0.1.0", + "langchain-openai>=0.2.0", + "langgraph>=0.2.0", + "supermemory>=0.1.0", + "nvidia-riva-client>=2.0.0", + "sounddevice>=0.4.6", + "numpy>=1.26.0", + "python-dotenv>=1.0.0", + "pydantic>=2.0.0", + "pydantic-settings>=2.0.0", + "rich>=13.0.0", + "typer>=0.9.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "ruff>=0.4.0", + "mypy>=1.10.0", +] +anthropic = [ + "langchain-anthropic>=0.3.0", +] + +[project.scripts] +thelab-chat = "thelab_langchain.cli:app" +thelab-bench = "benchmarks.runner:app" + +[project.urls] +Homepage = "https://github.com/derekclair/thelab" +Documentation = "https://supermemory.ai/docs/integrations/langchain" +Issues = "https://github.com/derekclair/thelab/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/thelab_langchain"] + +[tool.ruff] +line-length = 100 +target-version = "py311" +# `examples/` and `benchmarks/` are standalone illustrative scripts, not part of +# the importable library or the test surface, so they are excluded from linting. +extend-exclude = ["examples", "benchmarks"] + +[tool.ruff.lint] +select = ["E", "F", "I", "W", "UP"] +ignore = ["E501"] # line length handled by formatter + +[tool.mypy] +python_version = "3.11" +strict = true +warn_return_any = true +warn_unused_configs = true + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/specs/001-voice-dgx-spark-agent/plan.md b/specs/001-voice-dgx-spark-agent/plan.md new file mode 100644 index 0000000..21f5141 --- /dev/null +++ b/specs/001-voice-dgx-spark-agent/plan.md @@ -0,0 +1,241 @@ +# Technical Plan: Voice-Enabled Agent for NVIDIA DGX Spark (001) + +**Feature**: 001-voice-dgx-spark-agent +**Related Spec**: [spec.md](./spec.md) +**Date**: 2025-05-21 + +## 1. Architecture Overview + +We will build a **layered voice agent** with clear separation of concerns: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ User (Voice) │ +└───────────────────────┬─────────────────────────────────────┘ + │ Audio (mic/speaker) +┌───────────────────────▼─────────────────────────────────────┐ +│ Voice Layer (NVIDIA NeMo + Riva) │ +│ - Streaming ASR (STT) │ +│ - Streaming TTS │ +│ - VAD + Barge-in handling │ +└───────────────────────┬─────────────────────────────────────┘ + │ Text (transcribed utterance) +┌───────────────────────▼─────────────────────────────────────┐ +│ Orchestration Layer (Python) │ +│ - Conversation Manager / Turn Manager │ +│ - Interruption handling │ +│ - Audio ↔ Text bridging │ +└───────────────────────┬─────────────────────────────────────┘ + │ Structured input + context +┌───────────────────────▼─────────────────────────────────────┐ +│ Agent Brain (LangGraph) │ +│ - StateGraph with memory tools │ +│ - Supermemory (long-term) + short-term conversation state │ +│ - Tool calling (memory + future tools) │ +│ - LLM: Grok (default) or local model │ +└───────────────────────┬─────────────────────────────────────┘ + │ Tool calls / final response + ▼ + Supermemory + Other Tools +``` + +**Key Principle**: The voice layer is "dumb but fast". The LangGraph agent is the intelligent brain that decides *what* to say and *when* to use memory/tools. + +**Parallel lightweight spike (2026-06-12)**: A low-commitment hardware prototype loop lives in `conversational-voice-agent`: named-pipe (or Teams button) trigger, direct ALSA arecord/aplay on the specific Lenovo Go devices, `riva.client` streaming ASR with real partials, raw HID for the Teams LED, and a `speak()` / speak-pipe entry point for replies. It is used for rapid prototyping of the natural voice "feel" on real desk hardware and for exercising the *existing* agent + Supermemory (via the minimal seams) while the primary `thelab_langchain.voice` (orchestrator + audio + VAD) and telephony production path mature. See the local-tts handoff and its spike spec. The spike deliberately stays uncommitted to the final production architecture. + +**Serving Flexibility**: The LLM serving layer must be swappable. For v1, the target implementation uses the official NVIDIA NIM container (`nemotron-3-super-120b-a12b`) exposing an OpenAI-compatible endpoint at `http://localhost:8000/v1`. The agent is intended to use the existing `openai_compatible` provider for this deployment path. Future phases target clustered multi-node serving for full 340b-class models without changing the voice or agent code. + +**Target v1 Stack (Single-Node Desktop)**: +- **LLM**: Local ~30B Nemotron class via the `local-tts` / NeuTTS service (or NIM container for the 30B variant). Agent uses `openai_compatible` provider for the local endpoint. Grok fallback via xAI. +- Agent: LangGraph + Supermemory, configured to talk via `openai_compatible` for the NIM-based target deployment +- Voice: NVIDIA Riva / NeMo (ASR + TTS) via the existing VoiceOrchestrator + +## 2. Component Breakdown + +### 2.1 Voice Layer – NVIDIA NeMo / Riva + +**Recommended Stack**: +- **ASR (STT)**: NVIDIA NeMo Conformer or FastConformer (streaming) exported to TensorRT, served via **Riva ASR**. +- **TTS**: NVIDIA NeMo FastPitch + HiFi-GAN or RadTTS, served via **Riva TTS**. +- **Inference Server**: NVIDIA Riva (production-grade, gRPC + HTTP, excellent streaming support, optimized for DGX). +- **Alternative (lighter)**: Direct NeMo inference with TensorRT-LLM export if Riva overhead is undesirable for single-user setups. + +**Why Riva?** +- Best latency and concurrency characteristics on DGX Spark. +- Built-in support for word timestamps, confidence scores, and end-of-utterance detection. +- Official NVIDIA support and containers for DGX-class systems. + +**Integration Point**: +- The Python side will use the official `nvidia-riva-client` Python package (gRPC). +- We will implement an async audio streaming bridge. + +### 2.2 Orchestration Layer + +A new module `thelab_langchain.voice` (or `agent.voice`) responsible for: + +- Managing microphone input + speaker output (using `sounddevice` or `pyaudio`). +- Running Riva ASR in streaming mode and emitting final transcripts. +- Receiving text from the agent and feeding it to Riva TTS (with support for streaming synthesis). +- Handling barge-in (user speaking while TTS is playing → cancel current synthesis + notify agent). +- Turn management and simple VAD. + +This layer should feel like a "voice shell" around the existing agent. + +### 2.3 Agent Brain (LangGraph) + +We will evolve the current scaffolding in `src/thelab_langchain/agent/`: + +- Use a **custom StateGraph** (preferred over pure `create_react_agent` for better control over memory injection and voice-specific behaviors). +- Core nodes: + - `memory_injection` (pull relevant Supermemory context) + - `think` (LLM with tools bound) + - `act` (execute tools) + - `respond` (generate final utterance text) +- Tools will include the Supermemory tools we already started (`recall_memories`, `store_memory`, `get_user_profile`). +- The graph will be callable from the orchestration layer with a `user_id` + `thread_id`. + +**Important Design Decision**: +For voice, we want the agent to be able to produce **incremental / streaming text** so TTS can start early. This favors architectures where the LLM response is generated in chunks. + +### 2.3.1 Memory Integration Approach (Current Iteration) + +For the initial wiring of Supermemory: + +- Tools are created via `create_memory_tools(user_id)` factory (see `agent/tools/memory.py`) so they are properly scoped. This replaces the earlier global `_current_user_id` pattern. +- A `memory_injection` node will run early in the graph. It will: + - Call `get_user_profile(user_id)` to pull static facts + dynamic context. + - Call `recall_memories(query=last_user_utterance)` to surface the most relevant past memories. + - Inject the combined context into the system prompt or as a preceding message before the LLM step. +- In a follow-up pass, the three tools will be bound to the LLM via `.bind_tools()`, allowing the agent to proactively decide to call `store_memory(...)` or perform deeper recall. +- The `VoiceOrchestrator` will be responsible for creating the user-scoped tools and (later) passing them when constructing or invoking the graph. + +This two-phase approach (proactive injection first, reactive tool use second) gives immediate value while keeping the graph simple. + +**Current micro-iteration goals (both requested):** +- Improve the quality and conciseness of the injected memory context (better formatting + optional LLM summarization of retrieved memories). +- Begin binding the Supermemory tools to the LLM so the agent can call `store_memory`, `recall_memories`, or `get_user_profile` reactively when it decides it needs to. + +### 2.4 Memory Strategy + +- **Long-term**: Supermemory (via tools) — facts, preferences, project history, past conversations. +- **Short-term / Session**: LangGraph checkpointer (`MemorySaver` initially, later Postgres or Redis). +- The voice orchestration layer will pass the current `thread_id` on every turn. + +## 3. Packaging & Deployment (DGX Spark) + +### 3.1 Development & Deployment Workflow (Mac → DGX) + +**Primary flow** (as requested): + +1. **Develop locally** on Mac (this host). +2. **Dockerize** the application. +3. Build/push the image (or build directly on DGX). +4. On the DGX: `docker compose pull` (or build) → `docker compose up`. +5. Observe / interact with the running voice agent. + +This keeps the Mac as the fast development environment while the heavy GPU workloads (Riva + Nemotron inference) run on the DGX Spark. + +### 3.2 Docker Compose Architecture + +We will use a **multi-service `docker-compose.yml`** as the canonical way to run on DGX Spark. The stack will include at minimum: + +```yaml +services: + agent: # Our Python LangGraph + Supermemory voice agent + build: . + # ... GPU access, env vars, volumes ... + + riva: # NVIDIA Riva (ASR + TTS) + image: nvcr.io/nvidia/riva/riva-speech:... + # ... GPU, ports 50051 (gRPC), model volumes ... + + nemotron: # Local LLM brain (Nemotron NIM or vLLM) + image: nvcr.io/nim/nvidia/nemotron-... # or custom TRT-LLM engine + # Exposes OpenAI-compatible endpoint (so agent can use ChatOpenAI(base_url=...)) + # GPU resources allocated here +``` + +**Service Responsibilities**: +- **agent**: Runs our `thelab-langchain` code. Talks to: + - Riva (gRPC) for voice I/O + - Nemotron (HTTP, OpenAI-compatible) or Grok (via XAI) for reasoning + - Supermemory (cloud or future local) +- **riva**: Official NVIDIA Riva container(s) providing ASR and TTS. +- **nemotron**: NVIDIA NIM (or vLLM/TensorRT-LLM) serving a Nemotron model locally. This allows the agent to run fully on-DGX with no cloud LLM calls. + +The agent service will support runtime switching of the LLM backend via environment variables (e.g. `LLM_PROVIDER=grok` vs `LLM_PROVIDER=nemotron`). + +### 3.3 GPU & Resource Allocation on DGX Spark + +- Use `deploy.resources.reservations.devices` (or the older `runtime: nvidia`) to give containers access to the GPUs. +- Riva and Nemotron are both GPU-heavy; careful partitioning will be needed. +- The agent service itself is mostly CPU + light GPU usage (for any local post-processing). + +### 3.4 Model & Data Management + +- Large model artifacts (Riva models, Nemotron weights/engines) will live on named Docker volumes or host bind mounts (`/models`, `/data`). +- First-run download scripts or `docker compose run --rm model-downloader` targets. + +### 3.5 Observability Note + +Full production observability (Prometheus, Grafana, tracing, log aggregation) is deferred for now, as noted by the user. We will focus on basic structured logging and health endpoints initially. + +## 4. Implementation Phases + +**Phase 0** (Current spike) +- LangGraph + Supermemory tools foundation (already in progress on `langchain` branch) + +**Phase 1** – Core Voice Loop (MVP) +- Riva ASR + TTS integration (non-streaming first) +- Basic orchestration layer that can do: Listen → Transcribe → Agent → Synthesize → Speak +- CLI command: `thelab-chat voice` + +**Phase 2** – Streaming & Polish +- True streaming ASR + streaming TTS +- Barge-in support +- Better turn management and VAD +- Latency tuning on DGX Spark + +**Phase 3** – Productionization +- Docker + docker-compose optimized for DGX Spark +- Model caching and download scripts +- Health checks, logging, observability +- Optional local LLM backend (via vLLM + TensorRT-LLM) + +**Phase 4** (Future) +- Wake word +- Multi-speaker / voice cloning +- Full offline mode (local LLM + Supermemory local alternative if needed) + +## 5. Key Technical Risks & Mitigations + +| Risk | Mitigation | +|-----------------------------------|----------| +| High latency on voice round-trip | Prioritize streaming ASR + TTS from day one in Phase 2 | +| Riva complexity / container size | Provide both "full Riva" and "light NeMo" paths | +| Barge-in is hard | Use Riva's streaming endpoints + audio cancellation logic | +| GPU memory contention | Clear documentation + resource limits in compose file | +| Model download time on first run | Pre-download scripts + volume mounts | + +## 6. Interface Changes + +- New CLI entry: `thelab-chat voice --user derek --thread daily-standup` +- The existing text `thelab-chat chat` remains unchanged (great for debugging the brain without audio). +- The `MemoryChat` simple class can stay as a reference implementation. + +## 7. Success Metrics (Technical) + +- End-to-end voice latency (speech end → first audio out) < 2.5s on DGX Spark for normal queries. +- Accurate transcription of technical speech (thelab domain). +- Natural-sounding TTS that users actually enjoy listening to for long responses. +- Agent correctly uses long-term memory in voice conversations. + +## 8. Open Decisions to Resolve in Tasks + +- Exact NeMo model variants to start with (e.g., `stt_en_fastconformer_hybrid_large_streaming` + specific TTS voice). +- Whether to run full Riva server as a separate compose service or embed inference in the agent container. +- How much of the voice logic lives in pure Python vs calling into Riva SDK. + +--- + +**Next**: Generate `tasks.md` with dependency-ordered, actionable work items. Then begin implementation on the `langchain` branch. \ No newline at end of file diff --git a/specs/001-voice-dgx-spark-agent/spec.md b/specs/001-voice-dgx-spark-agent/spec.md new file mode 100644 index 0000000..c3350d5 --- /dev/null +++ b/specs/001-voice-dgx-spark-agent/spec.md @@ -0,0 +1,162 @@ +# Feature Spec: Voice-Enabled Agent for NVIDIA DGX Spark + +**Feature ID**: 001-voice-dgx-spark-agent +**Status**: Draft +**Created**: 2025-05-21 +**Owner**: Derek + +## Overview + +Build a production-grade, voice-first intelligent agent that runs locally on NVIDIA DGX Spark hardware. The system combines high-quality local speech-to-text (STT) and text-to-speech (TTS) with a powerful reasoning brain built on LangGraph + Supermemory + a local Nemotron model served via NVIDIA NIM (with Grok as a fallback option). + +The goal is a natural, low-latency voice conversation experience with long-term memory and tool use, packaged as a clean, deployable Docker-based agent. + +## Hardware Target & Scaling Strategy + +**Primary Target (v1)**: NVIDIA DGX Spark used as a **personal desktop AI workstation** — a machine sitting on the desk with physical microphone and speakers for natural, hands-free voice interaction. This is not a headless server deployment. The agent is intended to feel like a high-quality personal AI companion running locally on the device the user is working at. + +**Current Practical LLM**: On single-node DGX Spark, we target a responsive local model in the **~30B Nemotron class** (explicit requirement: no 120B+ models). Served via NVIDIA NIM or NeMo/NeuTTS local inference (see `conversational-voice-agent`). Grok (xAI) remains the convenient high-quality fallback when local is unavailable. + +**Future Scaling (Phase 2+)**: Acquire additional DGX Spark units (target 2–3 total) and cluster them to run the **full Nemotron 4 340b** (or equivalent 300B+ class model) at high quality while preserving the exact same voice, agent, and Supermemory experience. The voice + orchestration + LangGraph harness is deliberately designed to be portable across single-node efficient inference and multi-node clustered serving. + +**Why this approach?** +- The software architecture, voice experience, memory system, and tool use are the long-lived, hard parts. +- Raw inference scale (more GPUs, better model) can be added later once the harness proves valuable on real desktop workloads. +- This keeps iteration fast on a single machine while leaving a clear path to the highest-quality local model the user wants. + +## Goals + +- Deliver a high-quality, natural voice interface (mic + speakers) for the LangGraph + Supermemory agent on a personal DGX Spark desktop. +- Enable excellent local voice experience on single-node DGX Spark using the best practical local LLM that still feels responsive in conversation. +- Make the entire system easy to package, deploy, and run (Docker-first) as a personal desktop agent. +- Preserve (and enhance) the excellent long-term memory capabilities provided by Supermemory. +- Design the voice + agent harness so it can later scale to multi-node DGX Spark clusters running full 340b-class models without major changes to the upper layers. + +## Non-Goals (for v1) + +- Full multi-user / multi-tenant support +- Complex visual UI (terminal + voice is primary) +- Running the absolute largest possible model (340b-class) on a *single* DGX Spark in v1 — we accept the best responsive local model available (~120b-class) while designing the system to scale to 340b+ on clustered hardware later. + +## User Stories + +1. As a power user with a DGX Spark on my desk, I want to talk naturally to my personal agent (using the physical mic and speakers) so that I can have fluid, hands-free conversations while working. +2. As a developer, I want the agent to remember everything important about me and my projects across many sessions (via Supermemory). +3. As a user with sensitive data, I want the voice processing (STT/TTS) to happen locally on my DGX Spark so nothing leaves the machine except the reasoning calls I choose. +4. As an operator, I want to deploy the agent as a single Docker container (or small compose stack) on a personal DGX Spark desktop with minimal configuration. +5. As a long-term planner, I want the voice + agent architecture to be portable so that when I add more DGX Spark units I can run significantly larger local models (full 340b-class) without rewriting the upper layers. + +## Functional Requirements + +### FR-1: Voice Interface +- High-quality, low-latency Speech-to-Text using **NVIDIA NeMo** (Conformer / FastConformer family or Riva ASR). +- Natural, expressive Text-to-Speech using **NVIDIA NeMo** TTS models (FastPitch, RadTTS, or newer) served via Riva or optimized NeMo inference. +- Strong preference for streaming-capable ASR and TTS to minimize perceived latency. +- Support for push-to-talk and Voice Activity Detection (VAD) for natural turn-taking. +- Ability for the user to interrupt the agent mid-speech (barge-in support). + +### FR-2: Agent Brain (LangGraph + Supermemory) +- The existing LangGraph agent architecture (with Supermemory tools) becomes the core reasoning engine. +- The agent must be able to use tools while in a voice conversation. +- Long-term memory via Supermemory must remain first-class (profile + semantic recall + storage). +- Session persistence across restarts using thread IDs. + +### FR-3: Packaging & Deployment on DGX Spark +- The entire stack (STT + TTS + Agent + optional local LLM) must be containerized using Docker. +- Provide a `docker-compose.yml` (or equivalent) optimized for DGX Spark GPU usage. +- Clear documentation for running on DGX Spark hardware (GPU passthrough, model storage, etc.). +- Support for model caching / volume mounts for large STT/TTS models. + +### FR-4: Configuration & Extensibility +- Configuration via environment variables + `.env` (consistent with current project). +- Pluggable STT and TTS backends (easy to swap models). +- Option to route reasoning to either Grok (API) or a local LLM served via vLLM / Ollama / TensorRT-LLM on the DGX. + +### FR-5: Developer Experience +- The current CLI (`thelab-chat`) should evolve to support voice mode. +- Good logging and observability for the voice pipeline (latency, STT confidence, etc.). +- Health checks and graceful degradation if voice components fail. + +## Non-Functional Requirements + +- **Latency**: End-to-end voice turn (listen → think → speak) should feel responsive (< 2.5s target on DGX Spark class hardware for typical queries). +- **Resource Efficiency**: Must be able to run alongside other workloads on DGX Spark without monopolizing all GPUs. +- **Reliability**: STT/TTS failures should not crash the agent; graceful fallback to text mode. +- **Security**: Local STT/TTS means audio never leaves the machine unless explicitly sent to the reasoning LLM. +- **Maintainability**: Clear separation between voice layer and brain layer. + +## Success Criteria + +- User can have a natural back-and-forth voice conversation with the agent on DGX Spark. +- The agent correctly recalls and uses long-term memories from Supermemory during voice sessions. +- The system runs from a single `docker compose up` command (after initial model download). +- Switching between Grok and a local LLM for the brain is possible with minimal code/config changes. +- Audio quality is subjectively good (natural voice, accurate transcription). +- Button/DTMF test passes: Agent signals with a distinct new tone; correctly handles sequence of Microsoft Teams, Call Answer, Call End, Mute button presses with spoken continuity from user; mute is handled gracefully (hardware-level, no audio after). + +## Button / DTMF Test Scenario (Call Control) + +**Test Procedure** (to be executed in a telephony voice call with the local-tts service): + +1. Agent generates a distinct "new tone" (special audio signal via local-tts / NeuTTS or tone generator) to indicate readiness. +2. User pushes buttons in order and speaks after each: + - 1. Microsoft Teams button + - 2. Call Answer + - 3. Call End + - 4. Mute (suspected hardware mute — agent should detect loss of audio) +3. After each press, user speaks a short phrase for continuity. +4. Agent should detect DTMF tones or call events via the telephony provider and respond appropriately (e.g., acknowledge, handle call state changes, log for audit). + +**Requirements for local-tts + Voice Agent**: +- Support for DTMF detection and button event handling in the telephony webhook. +- Ability to generate distinct tones (new/custom tone for signaling). +- Graceful handling of mute (detect silence or a telephony call event, pause TTS, resume on unmute). +- Integration with a private VPN for secure webhook delivery. +- Observability via nv-monitor during the test (GPU/CPU during tone generation and call). + +This test validates call control, hardware button integration, and robustness of the local TTS service exposure. + +## Technology Decisions (Locked) + +**Voice Layer (STT + TTS)**: **NVIDIA NeMo** (with Riva inference stack where appropriate) is the chosen technology. + +Rationale: +- Native, best-in-class performance on DGX Spark / Blackwell hardware. +- Excellent support for low-latency streaming ASR and TTS. +- Consistent NVIDIA stack (easier optimization, TensorRT export, GPU utilization). +- Future-proofs us for fine-tuning custom voices or domain-specific ASR on the same hardware. + +We will use NeMo models (e.g., Conformer / FastConformer for ASR, FastPitch + HifiGAN or newer NeMo TTS models) served via NVIDIA Riva or direct NeMo inference optimized with TensorRT-LLM / ONNX Runtime. + +This decision was confirmed during spec review. + +**Spike / Prototype Vehicle note (2026-06-12)**: A working local voice I/O spike exists in `conversational-voice-agent` using the Lenovo Go Wired Speaker as the physical interface (mic + speaker + Teams button for trigger + LED for session feedback). It provides a rapid, low-commitment way to prototype the natural-language voice feel (named-pipe or button trigger, live partial transcripts via streaming NeMo ASR, spoken replies on the device, hardware LED/button feedback) while feeding the *existing* agent + Supermemory harness we already have. The seams are intentionally minimal (`send_partial_to_agent` for input to the brain, `speak()` / `/tmp/voice_speak` for output audio on the Lenovo Go) so the spike does not re-implement or bypass the agent brain / Supermemory. See `specs/001-interim-lenovo-go-voice-spike.md` in that repo. This is a prototyping vehicle and is intentionally uncommitted to final production voice layer choices (Riva vs. other, telephony integration, heavy models, the main VoiceOrchestrator path, etc.). + +## Related Work + +- Current spike lives on the `langchain` orphan branch (`src/thelab_langchain/agent/`) +- Supermemory + LangGraph memory tools already partially designed +- DGX Spark used as a personal desktop workstation (with physical voice I/O) is the primary target hardware for v1 +- Future multi-node DGX Spark clusters are the explicit scaling path for full 340b-class local models + +## Current Phase: Local High-Quality LLM on Single-Node DGX Spark (2026) + +**Decision**: For the initial desktop deployment on a single DGX Spark, we use the official NVIDIA NIM container for the ~120b Nemotron model (`nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b`) rather than raw Hugging Face weights served through vLLM or Docker Model Runner. + +**Rationale**: +- The official NIM provides a stable, well-supported OpenAI-compatible endpoint with good performance on DGX Spark. +- It proved more reliable in practice than Docker Model Runner + vLLM for this class of model on the current hardware. +- The agent already supports the `openai_compatible` LLM provider, making integration straightforward. +- This approach keeps the voice + agent harness unchanged while giving us a high-quality local brain today. + +**Endpoint**: The NIM exposes `http://localhost:8000/v1` (OpenAI chat completions compatible). + +**Trade-offs accepted**: +- We accept a small amount of vendor lock-in to NVIDIA's NIM packaging for the local model (acceptable given the hardware target). +- Docker Model Runner is retained for lighter experimentation and smaller models, but is no longer the primary path for the main reasoning model. + +This phase focuses on getting a rock-solid single-node experience with voice + 120b-class local LLM before scaling to multi-node 340b+. + +--- + +**Next**: After review, generate `plan.md` (technical architecture) and `tasks.md`. diff --git a/specs/001-voice-dgx-spark-agent/tasks.md b/specs/001-voice-dgx-spark-agent/tasks.md new file mode 100644 index 0000000..4e67c55 --- /dev/null +++ b/specs/001-voice-dgx-spark-agent/tasks.md @@ -0,0 +1,199 @@ +# Tasks: Voice-Enabled Agent for NVIDIA DGX Spark (001) + +**Feature**: 001-voice-dgx-spark-agent +|**Status**: Phase 2 implementation in progress (core wiring + local-tts 30B NeuTTS service + nv-monitor observability) +**Branch**: `spec/001-dgx-spark-voice-desktop-scaling` (implementation + docs evolution on the Phase 2 spec branch) + +This document breaks the plan into dependency-ordered, actionable tasks. Each task should be small enough to be completed in one focused session. + +--- + +## Phase 0 – Foundations (Current) + +- [x] Commit existing LangGraph + Supermemory tools scaffolding (`src/thelab_langchain/agent/`) +- [x] Create SDD artifacts (`spec.md`, `plan.md`, `tasks.md`) + +## Parallel Track A – Local High-Quality LLM + Voice on Single-Node DGX Spark Desktop + +**Goal**: Get a reliable, high-quality local ~30B Nemotron running on the desktop DGX Spark with full voice support via the dedicated `local-tts` NeuTTS service, while cleaning up previous experimental paths. The `local-tts` service is the canonical TTS provider exposed to the stack. + +**Current Status (as of 2026-06-12)**: Core agent + openai_compatible wiring + lazy voice imports complete and committed. `local-tts` 30B NeuTTS service initialized in dedicated repo with proper hygiene. nv-monitor observability tool integrated. Compose defaults to local-tts service. + +A lightweight **spike** in `conversational-voice-agent` (Lenovo Go hardware loop) has real streaming NeMo ASR (riva.client, live partials), Teams LED feedback (active during session), and partials wired to the agent seam. The spike adds a `speak()` + speak-pipe reply path and wires the physical Teams button to the (locked) trigger pipe so E2E (pipe/button → live partial STT → existing agent + Supermemory via seams → spoken reply on device) can be prototyped and "felt" on real hardware. See the local-tts handoff + its focused spike spec and the light note in this spec.md. The spike deliberately uses a lighter brain path (no 120B lockup) and does not re-implement the agent harness / Supermemory. + +Riva compatibility on current hardware and full production voice loop (telephony / main orchestrator) remain future items. + +### T2.1 – Local LLM Serving (NVIDIA NIM) +- [x] Document and stabilize running the official NIM container (`nemotron-3-super-120b-a12b`) on DGX Spark (direct `docker run` + compose service defaulting to 120b image) +- [x] Decide on long-term approach (Docker Model Runner abandoned for 120b+; direct NIM container is the path for v1 single-node desktop) +- [ ] Add convenience scripts or Makefile targets for starting/stopping the local NIM (future polish) + +### T2.2 – Agent Wiring to Local Endpoint +- [x] Make the agent (`MemoryChat` + CLI) cleanly support `openai_compatible` pointing at the local NIM (localhost:8000) via centralized `get_chat_model()` factory +- [x] Add sensible defaults / environment variable helpers for DGX Spark desktop use (.env.example block + compose defaults to 120b image + /opt/nim/.cache volume) +- [x] Update `thelab-chat env` / welcome screen to clearly show when using local model (Base URL display + openai_compatible warning) +- [ ] Ensure Supermemory + tool use works end-to-end with the local 120b model (pending full model load + test) + +### TA.3 – Voice + Local LLM Integration +- [ ] Verify / fix the `voice` command so it uses the same LLM path as `chat` +- [ ] Test full voice loop (ASR → agent with local LLM → TTS) on the desktop +- [ ] Handle cases where local LLM is slower (streaming, barge-in implications) + +### TA.4 – Cleanup & Code Health +- [ ] Reduce or remove reliance on the broken Docker Model Runner path for the main model +- [ ] Clean up related documentation, compose files, and old scripts +- [ ] Ensure all code changes are clean, well-tested where possible, and follow project conventions + +**Spike Track – local-tts Lenovo Go Voice Loop (interim hardware prototype for natural interface feel)** (added 2026-06-12; exercises the *existing* agent + Supermemory via minimal seams; deliberately lightweight and uncommitted to production choices): +- [x] Teams button light feedback implemented (led_control.py + integration in voice_loop; light stays active while session runs). +- [x] Real streaming NeMo ASR integrated using riva.client (replaced mock; partial transcripts flow in real time from first chunk). +- [x] Agent callback wiring: partial results sent to `send_partial_to_agent()` as soon as they arrive. +- [x] Named pipe trigger + 4 s window + LED during sessions (already working at spike start). +- [x] Implement bidirectional reply path (`speak(text)` using same riva.client style + `/tmp/voice_speak` pipe + daemon) with tone fallback. +- [x] Wire physical Teams button (evdev) to write "start" to the (locked) `/tmp/voice_trigger` pipe. +- [x] E2E test: named pipe (or button) + streaming STT (live partials) + agent seam + playback on the Lenovo Go speaker (with mock or real existing agent reply). +- [x] Update local-tts README + create focused `specs/001-interim-lenovo-go-voice-spike.md`. +- [x] Light sync note + status update + spike track checkboxes in thelab 001 (this file + spec.md + plan.md), emphasizing reuse of existing Supermemory-enabled agent. + +**Future Phase – Hardware Scaling (Post v1)**: Once the voice + agent harness is solid on a single DGX Spark desktop, add support for multi-node clustered inference to run full Nemotron 4 340b-class models while keeping the identical user experience. This will likely become its own follow-on spec (e.g. 007-multi-dgx-inference) that depends on 001. + +--- + +## Phase 1 – Core Voice Loop (MVP) + +### T1.1 – Project Structure & Dependencies +- [ ] Add new dependencies: `nvidia-riva-client`, `sounddevice`, `numpy`, `webrtcvad` or equivalent VAD +- [ ] Create `src/thelab_langchain/voice/` package +- [ ] Create `src/thelab_langchain/agent/graph.py` (initial StateGraph skeleton) +- [ ] Update `pyproject.toml` and `Makefile` as needed + +### T1.2 – Riva Client Wrapper +- [ ] Implement a clean async wrapper around `riva.client.ASRService` for streaming transcription +- [ ] Implement a clean async wrapper around `riva.client.TTSService` (initially non-streaming) +- [ ] Add configuration for Riva server address / port / SSL (even if running locally in same compose) + +### T1.3 – Basic Audio I/O +- [ ] Create microphone input loop using `sounddevice` +- [ ] Create speaker output using `sounddevice` +- [ ] Implement simple VAD to detect end of user utterance + +### T1.4 – Minimal Voice Orchestrator +- [ ] Build `VoiceOrchestrator` class that can: + - Listen until end of speech + - Get transcript from Riva ASR + - Send text to the LangGraph agent + - Receive response text + - Synthesize with Riva TTS and play audio +- [ ] Wire it to the existing `MemoryChat` or new graph as a first integration test + +### T1.5 – CLI Voice Command +- [ ] Add `thelab-chat voice` subcommand (using Typer) +- [ ] Support `--user` and `--thread` flags +- [ ] Basic error handling and fallback to text mode if audio devices fail + +--- + +## Phase 2 – Streaming & Natural Conversation + +### T2.1 – Streaming ASR +- [ ] Switch ASR integration to true streaming mode (incremental transcripts + final) +- [ ] Implement partial transcript handling (optional: show live transcription) + +### T2.2 – Streaming TTS + Early Audio +- [ ] Upgrade TTS to streaming synthesis (Riva supports chunked audio) +- [ ] Start playing audio as soon as first TTS chunks arrive (reduces perceived latency) + +### T2.3 – Barge-in Support +- [ ] Detect user speech while TTS is playing +- [ ] Cancel current TTS synthesis +- [ ] Notify the agent that the previous response was interrupted +- [ ] Allow the agent to react appropriately ("Sorry, what were you saying?" or just listen) + +### T2.4 – Improved Turn Management +- [ ] Robust state machine for conversation turns (Listening / Thinking / Speaking / Interrupted) +- [ ] Better handling of overlapping speech + +--- + +## Phase 3 – Packaging & DGX Spark Deployment + +### T3.1 – Docker Compose Stack (Mac → DGX Workflow) +- [ ] Create `Dockerfile` for the `agent` service (Python + our code + Riva client) +- [ ] Create root `docker-compose.yml` with at least three services: + - `agent` (our voice + LangGraph app) + - `riva` (official NVIDIA Riva container) + - `nemotron` (NVIDIA NIM or vLLM serving Nemotron model, OpenAI-compatible) +- [ ] Support `docker compose --profile full-riva` or similar for different deployment sizes +- [ ] Add `.env.example` with all required variables for the compose stack (including `NEMOTRON_BASE_URL`, `RIVA_URI`, etc.) +- [ ] Document the "Develop on Mac → docker build/push → pull & `docker compose up` on DGX" workflow + +### T3.2 – Nemotron Integration (Local LLM Brain) +- [ ] Make the agent able to use a local Nemotron NIM as drop-in replacement for Grok + - Use `langchain-openai.ChatOpenAI` with custom `base_url` + `api_key` +- [ ] Add environment variable switching (`LLM_PROVIDER=nemotron` vs `grok`) +- [ ] Document recommended Nemotron models for DGX Spark (size vs quality tradeoffs) +- [ ] Create a small health-check / smoke test that the Nemotron endpoint is reachable from the agent service + +### T3.3 – Riva as Separate Service +- [ ] Wire the agent container to talk to the `riva` service over the Docker network (usually `riva:50051`) +- [ ] Provide volume strategy for Riva model cache +- [ ] Document how to start just Riva for development/testing + +### T3.4 – DGX Spark Documentation & Tooling +- [ ] Write `docs/dgx-spark-deployment.md` +- [ ] Cover: NVIDIA Container Toolkit, GPU device requests in compose, running alongside other workloads, SSH access patterns (`ssh dgx`) +- [ ] Add helper scripts (e.g. `scripts/push-to-dgx.sh` or instructions using `docker buildx` / registry) + +### T3.3 – DGX Spark Documentation +- [ ] Write `docs/dgx-spark-deployment.md` +- [ ] Cover: NVIDIA Container Toolkit, GPU visibility, performance tuning, running alongside other workloads + +### T3.4 – Health & Observability +- [ ] Add health endpoints (FastAPI or simple HTTP) for the agent +- [ ] Structured logging for voice pipeline latency (STT time, agent time, TTS time) +- [ ] Graceful degradation when voice hardware is unavailable + +--- + +## Phase 4 – Agent Brain Evolution + +### T4.1 – Proper LangGraph Agent +- [ ] Replace simple `MemoryChat` usage with a real `StateGraph` +- [ ] Implement memory injection node that calls Supermemory tools before thinking +- [ ] Bind the memory tools to the LLM +- [ ] Support `thread_id` + `checkpointer` + +### T4.2 – Voice-Aware Behaviors +- [ ] Add special instructions / system prompt variations for voice (shorter responses, more conversational tone) +- [ ] Handle "interrupted" signals from the orchestration layer inside the graph + +### T4.3 – Tool Surface Expansion (optional but recommended) +- [ ] At minimum keep the three Supermemory tools solid +- [ ] Consider adding a simple "clarify" tool or confirmation pattern for voice + +--- + +## Cross-Cutting / Polish Tasks + +- [ ] Comprehensive error handling and user-friendly messages in voice mode +- [ ] Configuration system for choosing between Riva and direct NeMo +- [ ] Unit + integration tests for the voice layer (mocked Riva) +- [ ] Update main README with voice capabilities and DGX Spark section +- [ ] Performance profiling run on actual DGX Spark hardware (latency numbers) + +--- + +## Suggested Order of Implementation + +1. **T1.1 → T1.4** (Get something that can talk and listen, even if clunky) +2. **T1.5** (Make it usable via CLI) +3. **T2.1 – T2.4** (Make it feel natural – this is where the magic happens) +4. **T4.1 – T4.2** (Upgrade the brain to a real graph while doing voice work) +5. **T3.1 – T3.4** (Packaging – do this once the core loop is stable) +6. Polish + documentation + +--- + +**Ready to start?** The first concrete task is usually **T1.1 – Project Structure & Dependencies**. + +Let me know when you want to begin implementation (I can start with T1.1 right now if desired). diff --git a/specs/002-multi-user-support/spec.md b/specs/002-multi-user-support/spec.md new file mode 100644 index 0000000..6af56d1 --- /dev/null +++ b/specs/002-multi-user-support/spec.md @@ -0,0 +1,98 @@ +# Feature Spec: Multi-User Support for the Voice Agent + +**Feature ID**: 002-multi-user-support +**Status**: Draft / Future +**Related to**: [001-voice-dgx-spark-agent](../001-voice-dgx-spark-agent/spec.md) +**Created**: 2025-05-21 + +## Overview + +The voice agent should gracefully support multiple users within the same household (initially: Derek, wife, two daughters, son, and occasional "other" guests or family members). + +Each person should have their own persistent identity and long-term memory context. When someone speaks to the agent, it should correctly identify who they are (or be told) and recall the right history, preferences, ongoing projects, and relationships. + +This is a **cross-cutting concern** that affects user identification, Supermemory container isolation, session/thread management, the LangGraph state, and the overall voice experience. + +## Goals + +- Natural multi-user experience in a family setting. +- Strong long-term memory isolation per person (via Supermemory `container_tag`). +- Reasonable accuracy in knowing "who is talking" without constant re-identification. +- Future-proof for adding more family members or occasional guests. +- Maintain privacy boundaries between users. + +## Non-Goals (for initial version) + +- Full biometric voice fingerprinting / speaker diarization (nice to have later). +- Remote multi-user access from outside the home. +- Complex household roles/permissions system. +- Guest accounts with temporary memory. + +## User Stories + +1. **As Derek**, I want the agent to remember my ongoing projects, preferences, and conversations even when other family members have spoken to it recently. +2. **As my wife**, I want the agent to remember things that are important to me (kids' schedules, our shared tasks, etc.) without mixing them up with Derek's work stuff. +3. **As a kid**, I want the agent to know who I am when I talk to it and remember things like my homework, favorite games, or ongoing stories. +4. **As a parent**, I want to be able to say "Hey Lab, this is Sarah talking" or have the system figure it out reasonably well. +5. **As the household**, we want the agent to understand family relationships ("my sister", "Dad", "the kids") when context is relevant. + +## Functional Requirements + +### FR-1: User Identity & Routing +- The system must be able to associate a voice interaction with a specific user identity. +- Supported identification methods (in rough priority order): + 1. Explicit declaration ("Hey Lab, it's Derek") + 2. Wake-word + name patterns + 3. Heuristic / voice characteristics (future) + 4. Device or room context (if multiple microphones are added later) + +### FR-2: Memory Isolation (Supermemory) +- Every user must have their own `container_tag` in Supermemory. +- All `profile()`, `add()`, and `search` calls must be correctly scoped to the identified user. +- Cross-user leakage must be prevented (the agent should not accidentally recall one person's private facts to another). + +### FR-3: Session & Thread Management +- Each user should have their own conversation threads (`thread_id`). +- Short-term memory (LangGraph checkpointer) must be isolated per user. +- It should be possible to have parallel conversations with different family members. + +### FR-4: Relationship & Household Context +- The agent should be able to reason about family relationships when given the right context ("Tell my wife...", "What does Dad usually say about this?"). +- There may be a lightweight "household" or "family" memory layer in addition to individual profiles. + +### FR-5: Graceful Handling of Unknown Speakers +- If the agent cannot confidently identify the speaker, it should ask for clarification in a friendly way ("Sorry, I didn't catch who I'm speaking with — is this Derek, Sarah, or one of the kids?"). + +## Non-Functional Requirements + +- **Privacy**: One family member's private memories or conversations must never leak to another. +- **Low Friction**: Identification should feel natural, not like logging into a system every time. +- **Scalability**: Design should support adding more users without major rewrites. +- **Auditability** (future): It should be possible to see which user a memory belongs to. + +## Open Questions + +- How do we initially bootstrap user identities? (Manual config file? First-time "register yourself" flow?) +- Should there be a concept of a "primary user" (Derek) who has elevated capabilities? +- Do we want speaker diarization / voice embedding models running locally on the DGX for passive identification? +- How do we handle "other" / guests? Temporary containers? A generic "guest" profile? +- Should the agent proactively learn voices over time ("You sound like Maya today")? + +## Relationship to Feature 001 + +This feature is a natural evolution of the single-user voice agent defined in 001. + +The core architecture decisions made in 001 (Supermemory `container_tag` per user, `thread_id` per session, LangGraph state) were intentionally designed to be multi-user friendly. This spec captures the additional work needed to make the experience truly multi-user in a family context. + +## Success Criteria (for when we eventually implement) + +- Derek, his wife, and both kids can have natural, separate ongoing conversations with the agent over weeks/months with correct memory recall. +- The agent rarely confuses one person's context with another's. +- Adding a new family member is a low-effort configuration task. +- The experience feels personal and "knows" each person without feeling creepy or overly technical. + +--- + +**Status**: This spec is captured for future planning. It is **not** in scope for the current implementation wave. + +Next time we pick up multi-user work, we should create a `plan.md` and `tasks.md` under this directory following the established SDD process. \ No newline at end of file diff --git a/specs/003-deployment-infrastructure/spec.md b/specs/003-deployment-infrastructure/spec.md new file mode 100644 index 0000000..dd5b115 --- /dev/null +++ b/specs/003-deployment-infrastructure/spec.md @@ -0,0 +1,109 @@ +# Feature Spec: Deployment Infrastructure & Dockerization (Gaps & Improvements) + +**Feature ID**: 003-deployment-infrastructure +**Status**: Draft +**Related**: 001-voice-dgx-spark-agent +**Date**: 2025-05-21 + +## Overview + +The current Docker implementation (`Dockerfile` + `docker-compose.yml`) provides a basic skeleton for running the voice agent stack on DGX Spark. However, it has several gaps that would make real-world development, deployment, reliability, and operations painful — especially in a multi-GPU DGX environment shared with other workloads. + +This spec captures the identified gaps and defines the target state for a production-capable deployment infrastructure. + +## Current State (as of May 2025) + +- Basic `Dockerfile` that installs the Python package in a slim image. +- `docker-compose.yml` with three services: `agent`, `riva`, `nemotron`. +- High-level deployment story documented in `plan.md` (Mac dev → build/push → DGX `docker compose up`). +- No healthchecks, limited resource controls, minimal volume strategy, no secrets management, weak networking documentation. + +## Identified Gaps + +### 1. Reliability & Operations +- No healthchecks on any service. +- No restart policies or restart backoff configuration. +- No readiness/liveness probes that the agent can use (especially important when Riva or Nemotron are still loading models). +- No graceful shutdown handling. + +### 2. Resource Management on DGX Spark +- No explicit GPU device requests or limits per service (critical when multiple heavy services run on the same box). +- No CPU/memory limits. +- No support for different GPU allocation profiles (e.g., "light" vs "full" inference). + +### 3. Model & Data Management +- Model volumes are declared but there is no clear strategy for: + - First-time model download / initialization + - Model versioning + - Sharing models across containers efficiently + - Backup/restore of important user data + +### 4. Networking & Service Discovery +- Hardcoded service names (`riva`, `nemotron`) assume they are on the same Docker network. +- No clear documentation of required ports and protocols. +- No support for running the agent in "local dev" mode (talking to host-run Riva/Nemotron) vs full compose stack. + +### 5. Configuration & Secrets +- Environment variables are passed through but there is no `.env` template or validation at compose time. +- No distinction between required vs optional variables. +- API keys (Supermemory, XAI, etc.) are treated as plain env vars with no secrets management story for DGX. + +### 6. Image Build & Distribution +- No multi-stage optimization for smaller runtime images. +- No image tagging strategy (e.g., `git-sha`, `latest`, versioned releases). +- No documented path for using a private container registry (as the user is currently acquiring credentials for). + +### 7. Development Experience +- Difficult to run just the agent code locally with mocked voice/LLM services. +- No `docker-compose.override.yml` or profiles for local development vs DGX production. +- No easy way to run tests inside the container. + +### 8. Observability (Deferred per prior decision) +- No logging configuration. +- No metrics or tracing hooks (explicitly kicked for now). + +## Requirements + +- The deployment must be reliable enough to survive model loading delays and occasional GPU OOM situations on DGX. +- It must be possible for a new engineer (or future agent) to get the full stack running on a DGX Spark with reasonable effort. +- The same image built on a Mac (or CI) must run correctly on DGX. +- Clear separation between "I just want to hack on the agent logic" and "I want the full voice stack with real Riva + Nemotron". +- Support for the user's desired workflow: develop locally → dockerize → push to private registry → pull & run on DGX. + +## Proposed Improvements (Prioritized) + +### Phase A (High Value, Low Risk) +- Add healthchecks to all services. +- Add proper `deploy.resources` GPU reservations in compose. +- Create `.env.example` + validation. +- Improve `Dockerfile` (multi-stage, non-root user, smaller layers, better caching). +- Add Docker Compose profiles (`dev`, `full`, `riva-only`, etc.). +- Document local development with mocked services. + +### Phase B (Medium) +- Robust volume strategy + model downloader helper service or script. +- Private registry push/pull workflow + credentials handling. +- Restart policies + resource limits. +- Basic structured logging configuration. + +### Phase C (Later) +- Secrets management (Docker secrets, 1Password, Vault, etc.). +- Observability stack (when the team is ready). +- Automated image builds in CI. + +## Success Criteria + +- A new person can follow documented steps and have the full voice agent running on a DGX Spark in under 2 hours (assuming models are pre-cached or download is acceptable). +- The agent container can be updated independently of the heavy inference services. +- Switching between Grok and local Nemotron is a one-line environment variable change (already partially achieved). +- The stack is resilient to individual service restarts. + +## Out of Scope (for this spec) + +- Full production Kubernetes / DGX-specific orchestration (future). +- Advanced canary / blue-green deployments. +- Cost optimization across multiple DGX nodes. + +--- + +**Next Steps (when we pick this up)**: Create `tasks.md` under this directory and begin closing the highest-impact gaps (healthchecks, resource requests, dev experience, registry workflow). \ No newline at end of file diff --git a/specs/004-persistence-checkpointers/spec.md b/specs/004-persistence-checkpointers/spec.md new file mode 100644 index 0000000..7d7396f --- /dev/null +++ b/specs/004-persistence-checkpointers/spec.md @@ -0,0 +1,65 @@ +# Feature Spec: Persistence & Checkpointers + +**Feature ID**: 004-persistence-checkpointers +**Status**: Draft +**Related**: 001-voice-dgx-spark-agent +**Date**: 2025-05-21 + +## Overview + +The current LangGraph agent has no persistent checkpointer. All short-term conversation state lives only in memory. If the agent container restarts (or the DGX is rebooted), all in-progress conversations and recent context are lost. + +For a voice agent that family members will talk to over days and weeks, we need reliable short-term memory persistence in addition to the long-term Supermemory store. + +## Goals + +- Conversations survive agent restarts and DGX reboots. +- Multiple family members can have independent, persistent threads. +- Easy to swap between different checkpointer backends (in-memory for dev, Postgres/Redis for production). +- Reasonable performance on DGX Spark. + +## Current State + +- Graph is built with `compile()` but no `checkpointer` argument is passed. +- `VoiceOrchestrator` passes `thread_id` but it is only used for logging / Supermemory `container_tag`. +- LangGraph's `MemorySaver` (in-memory) is the implicit default. + +## Requirements + +- The checkpointer must support the standard LangGraph checkpoint interface (`get`, `put`, `list`). +- It must be possible to configure the backend via environment variables. +- Thread IDs must be unique per user (`{user_id}:{thread_id}` or similar namespacing). +- The solution must work both locally (Mac dev) and on DGX Spark. + +## Recommended Options + +1. **In-memory** (`MemorySaver`) – great for local development and quick tests. +2. **SQLite** (via `langgraph-checkpoint-sqlite`) – simple, file-based, zero dependencies, good enough for single-DGX use. +3. **PostgreSQL** (via `langgraph-checkpoint-postgres`) – proper production choice, supports multiple agents, good concurrency. +4. **Redis** – fast, but more operational overhead. + +For a home/DGX Spark deployment, **SQLite** is an excellent pragmatic default, with Postgres as an easy upgrade path. + +## Proposed Design + +- Add a `CHECKPOINTER_BACKEND` env var (`memory`, `sqlite`, `postgres`). +- Create a small factory `get_checkpointer()` in the agent package. +- When building the graph in `get_agent(user_id=...)`, pass the checkpointer. +- Namespacing strategy: use `{user_id}::{thread_id}` as the thread ID passed to LangGraph so isolation is natural. +- Store the SQLite file on a persistent Docker volume so it survives container restarts. + +## Open Questions + +- Should we also persist the most recent voice context / audio state? (Probably not — keep it lightweight.) +- Do we want automatic checkpoint cleanup (e.g., keep last N checkpoints per thread)? +- How do we handle migration if we change checkpointer backends later? + +## Success Criteria + +- Restarting the `agent` container does not lose active conversation threads. +- Different family members can interleave conversations without state corruption. +- Switching from SQLite to Postgres is a one-line config change + volume migration. + +--- + +**Next Steps (when picked up)**: Create `tasks.md`, implement the checkpointer factory, wire it into `get_agent()`, update compose to mount a persistent volume for SQLite, and add basic documentation. \ No newline at end of file diff --git a/specs/005-testing-and-cicd/spec.md b/specs/005-testing-and-cicd/spec.md new file mode 100644 index 0000000..da780d4 --- /dev/null +++ b/specs/005-testing-and-cicd/spec.md @@ -0,0 +1,92 @@ +# Feature Spec: Testing Strategy, CI/CD, and Coverage + +**Feature ID**: 005-testing-and-cicd +**Status**: Draft +**Related**: 001-voice-dgx-spark-agent, 003-deployment-infrastructure +**Date**: 2025-05-21 + +## Overview + +The repository currently has almost no automated tests, no CI pipeline, and no coverage measurement. As the system grows (especially the agent brain, voice layer, and multi-service Docker stack), this becomes a major risk. + +This spec defines the target testing and delivery infrastructure. + +## Current State + +- No `tests/` directory with meaningful coverage. +- No `pytest` configuration beyond a stub in `pyproject.toml`. +- No GitHub Actions or other CI workflow. +- No coverage reporting (codecov, etc.). +- Manual testing is the primary validation method. + +## Goals + +- Confidence that changes to the agent graph, tools, or voice layer do not break existing behavior. +- Fast feedback on pull requests. +- Ability to safely evolve the system on DGX Spark without constant manual smoke testing. +- Reasonable coverage targets without slowing down development. + +## Testing Strategy + +### Unit Tests +- Pure logic: config loading, LLM factory, memory tool helpers, prompt construction, state reducers. +- Mocked external services (Supermemory client, Riva client, LLM calls). +- Target: fast (< 30s total suite). + +### Integration Tests +- Graph execution with real (or lightly mocked) tools. +- End-to-end voice loop with mocked audio/Riva (record → ASR → agent → TTS → play). +- Docker Compose smoke tests (does the stack start and report healthy?). +- These can be slower and may require GPU or specific services. + +### Contract / Golden Tests (future) +- Snapshot testing of memory injection output for known user contexts. +- Regression tests for prompt formatting. + +## CI/CD Pipeline (Proposed) + +**Platform**: GitHub Actions (standard for this repo style) + +**Workflows**: +1. **CI** (on push/PR to main and feature branches) + - Lint + type check (`ruff`, `mypy`) + - Unit tests + coverage + - Build Docker image (multi-platform if needed) + - Optional: integration tests (can be gated) + +2. **Docker Build & Push** (on tags or manual) + - Build agent image + - Push to private container registry (user is acquiring credentials) + - Tag with `git-sha` and semver when appropriate + +3. **Deployment** (manual or protected branch) + - Trigger on DGX (via webhook, SSH, or ArgoCD-style in the future) + +**Coverage Target** (initial): +- Minimum 60% overall, with higher expectations on core agent logic (80%+ on `agent/` and `voice/` packages). +- Fail PRs only on significant drops, not on every new file. + +## Tooling Recommendations + +- **pytest** + `pytest-asyncio` (already partially declared) +- **coverage.py** or `pytest-cov` +- **ruff** + **mypy** (already in dev deps) +- GitHub Actions cache for pip and Docker layers +- Optional: `act` for local CI testing + +## Out of Scope (initial wave) + +- End-to-end hardware-in-the-loop tests on actual DGX (too slow/expensive for every PR) +- Mutation testing +- Performance benchmarking in CI + +## Success Criteria + +- Any developer can run the full test suite locally with `make test` (or `pytest`). +- Every PR gets automated feedback on lint, types, and coverage. +- We can confidently cut releases and push new agent images to the DGX registry. +- New contributors (or future agents) can understand how to add tests by looking at existing examples. + +--- + +**Next Steps (when picked up)**: Create `tasks.md`, set up initial `tests/` structure + pytest config, add GitHub Actions workflow, wire coverage reporting, and update `Makefile`. \ No newline at end of file diff --git a/specs/006-alternative-memory-systems/spec.md b/specs/006-alternative-memory-systems/spec.md new file mode 100644 index 0000000..680df06 --- /dev/null +++ b/specs/006-alternative-memory-systems/spec.md @@ -0,0 +1,61 @@ +# Spec: Alternative Long-Term Memory Backends + +**Feature ID**: 006-alternative-memory-systems +**Status**: Draft / Future Consideration +**Related**: 001-voice-dgx-spark-agent, 002-multi-user-support +**Date**: 2025-05-21 + +## Motivation + +Supermemory is currently our primary long-term memory store. It provides excellent automatic profiling and semantic search. However, there are reasons we may want to support (or migrate to) other systems in the future: + +- Cost / latency (cloud round-trips) +- Privacy / air-gapped DGX deployments +- Different retrieval characteristics (graph memory, temporal, hierarchical, etc.) +- Vendor risk / lock-in + +## Current Usage Pattern + +All long-term memory access goes through three tools: + +- `get_user_profile()` +- `recall_memories(query, limit)` +- `store_memory(content, metadata)` + +These are the only places that talk to the underlying memory system. The rest of the agent (graph nodes, voice layer) is decoupled from the specific backend. + +This is intentional and makes swapping backends relatively cheap. + +## Candidate Alternative Systems + +| System | Strengths | Weaknesses | Fit for DGX Voice Agent | +|---------------------|----------------------------------------|-------------------------------------|-------------------------| +| **Zep** | Strong session + long-term memory, good SDK | Cloud-first, newer | Good | +| **Mem0** | Lightweight, self-hostable, user profiles | Less mature profiling than Supermemory | Promising | +| **LangGraph Memory**| Native checkpoint + long-term stores | Still evolving | Natural for this stack | +| **Custom Vector + Graph** | Full control, can run entirely locally | High implementation cost | Possible long-term | +| **SQLite + embeddings** | Dead simple, local, no extra services | Limited recall quality | Good for very constrained setups | + +## Proposed Approach (when we decide to invest) + +1. Define a small `MemoryBackend` protocol / abstract base class with the three methods we actually use. +2. Implement adapters for the systems we care about. +3. Make the tool factory (`create_memory_tools`) pluggable so different users or deployments can choose different backends. +4. Keep Supermemory as the default for the family voice agent (excellent UX today). + +## When This Becomes Relevant + +- We want a fully air-gapped DGX deployment (no outbound Supermemory calls). +- Cost of Supermemory becomes material at household scale. +- We discover specific recall quality problems that another system solves better. +- We want graph-based reasoning over memories (e.g., "who in the family knows about X?"). + +## Recommendation + +Do **not** build this yet. The current Supermemory integration (with proactive injection + reactive tools) is already delivering strong value. + +Treat this as an architectural "escape hatch" that we have deliberately kept open by using a narrow tool interface. + +--- + +**When we pick this up**: Create tasks, define the `MemoryBackend` interface, and implement the first alternative adapter (probably Mem0 or a simple local vector store). \ No newline at end of file diff --git a/specs/007-dgx-hardware-optimization/plan.md b/specs/007-dgx-hardware-optimization/plan.md new file mode 100644 index 0000000..4b2e6f5 --- /dev/null +++ b/specs/007-dgx-hardware-optimization/plan.md @@ -0,0 +1,152 @@ +# Technical Plan: DGX Spark Hardware Optimization & Sweet-Spot Discovery (007) + +**Feature**: 007-dgx-hardware-optimization +**Related Spec**: [spec.md](./spec.md) +**Date**: 2025-05-22 +**Implementation Branch**: `feat/007-dgx-hardware-optimization-impl` + +## 1. Goal + +Execute the strategy defined in the spec with rigorous measurement: + +- Capture an accurate **baseline** on the current production configuration (120B + full Riva on single DGX Spark). +- Run controlled experiments for the highest-leverage changes (lightweight English audio stack, 49B model swap). +- Quantify headroom, voice turn latency, concurrency limits, and memory behavior. +- Make a data-driven decision on the **sweet-spot configuration**. +- Document everything so future changes (including 2× node work) have a clear before/after reference. + +Success = we have reproducible numbers and a locked "recommended daily driver" profile for the family voice agent. + +## 2. High-Level Phases + +### Phase 0 – Baseline Capture (Must Do First) +Establish the "as-is" numbers on the exact current stack before touching anything. + +- Run on clean DGX Spark with current `docker-compose.yml` + nemotron-3-super-120b-a12b + full Riva. +- Instrument or manually measure the core metrics from the spec. +- Produce a `baseline-report.md` (or JSON + human summary) committed in the repo. + +### Phase 1 – Audio Stack Reduction (Highest Leverage Quick Win) +Replace full Riva with a minimal English-only path (Parakeet CTC + high-quality English TTS NIM or equivalent). + +- Create a lightweight audio service profile (new container or slimmed Riva config). +- Update `docker-compose` with profiles or separate override files. +- Re-run the benchmark harness. +- Compare delta vs baseline (memory saved, latency change, perceived voice quality). + +**Decision gate**: If quality is acceptable and headroom improves significantly → adopt as new default. + +### Phase 2 – Model A/B Testing (49B vs Current 120B) +Stand up `llama-3.3-nemotron-super-49b-v1.5` alongside the 120B. + +- Add a second LLM service in compose (different port or profile). +- Make the agent configurable (env var or CLI flag) to point at different NIM endpoints. +- Run identical benchmark scenarios on both models (with the winning audio stack from Phase 1). +- Measure: latency (especially TTFT + full turn), memory headroom, subjective quality on memory-recall + household prompts, tool-calling reliability. + +**Decision gate**: Choose primary model (likely 49B for daily use, 120B as optional "deep" mode). + +### Phase 3 – Context & Memory Efficiency Tuning +With the chosen model + audio, optimize how we use the remaining headroom. + +- Improve / implement aggressive yet high-quality summarization of long conversations before injection. +- Tune active context window size vs. KV cache cost. +- Measure impact on recall quality (via Supermemory) vs. memory usage and latency. +- Validate multi-user (2–4 concurrent simulated family members) stability. + +### Phase 4 – Sweet-Spot Lock + Operationalization +- Update default `docker-compose.yml`, `.env` examples, and Makefile targets for the chosen configuration. +- Add documented "benchmark" and "profile" make targets. +- Update architecture docs and the 001 spec references. +- Create a "current sweet spot" section in the 007 directory with the final numbers and rationale. + +### Phase 5 – 2× DGX Spark Preparation (Future, After Phase 4) +- Design multi-node compose / orchestration approach (tensor-parallel for 340B or service separation). +- Document networking (RDMA) requirements and expected gains. +- Optional: small spike to validate 2-node connectivity and basic sharding. + +## 3. Benchmark Harness Design + +We will build a lightweight, reproducible measurement system. + +### 3.1 Core Components +- `benchmarks/` directory at repo root (or inside `scripts/benchmarks/`). +- `benchmark_runner.py` (or Typer CLI `thelab-bench`). +- Scenarios: + - Single-user short turns (typical Q&A + memory recall). + - Multi-turn long-context household conversation. + - Concurrent simulation (2–4 "users" via scripted or parallel processes). +- Metrics collection: + - Voice turn latency (instrumented in VoiceOrchestrator or via external timing around the full loop). + - LLM TTFT + generation speed (via callbacks or NIM metrics if exposed). + - Peak / average memory (system + per-container via `docker stats`, `nvidia-smi`, or `psutil` + CUDA). + - Error / OOM / swap events. + - Thermals / power (optional, via `tegrastats` or similar on Grace). + +### 3.2 Instrumentation Points (Temporary or Permanent) +- Add timing hooks in `src/thelab_langchain/voice/orchestrator.py` (end-of-speech → transcript ready → agent response start → first audio out). +- Expose a `--benchmark` mode that logs structured JSON lines. +- Sidecar measurement script that samples memory every 1–2 seconds during a run. + +### 3.3 Reporting +- Each run produces a timestamped report directory: `benchmarks/reports/2025-05-22-baseline/` +- Contains: `metrics.json`, `summary.md`, raw logs, `nvidia-smi.log`, container stats. +- A small script to generate comparison tables between two reports. + +### 3.4 Make Targets (for DX on DGX and Mac) +- `make benchmark-baseline` +- `make benchmark-light-audio` +- `make benchmark-49b` +- `make benchmark-report COMPARE=baseline,light-audio` + +These will be thin wrappers that set the right compose profiles + env and invoke the harness. + +## 4. Docker & Deployment Changes + +- Keep the existing `docker-compose.yml` as the "current baseline" reference. +- Introduce compose profiles or override files: + - `docker-compose.light-audio.yml` + - `docker-compose.49b.yml` + - Later: `docker-compose.2node.yml` (or separate stack) +- Make the LLM service tag and Riva vs light-audio configurable via environment (REGISTRY + MODEL_TAG + AUDIO_PROFILE). +- Ensure non-root, healthchecks, and easy `docker compose --profile` usage remain. + +The agent code itself should require **minimal** changes for the experiments (mostly config + endpoint URLs). + +## 5. Data-Driven Decision Process + +After each major phase we will: +1. Run the benchmark harness (at least 3–5 representative sessions). +2. Commit the raw report + a human-readable `results-phase-N.md`. +3. Update the decision matrix in the spec (or a living `decision-log.md`). +4. Hold a quick "gate" discussion (even async via PR comment or the issue tracker) before proceeding to the next phase. + +No optimization change lands in the default compose without passing through this measured gate. + +## 6. Risks & Mitigations + +- **DGX access / iteration speed**: All heavy runs happen on the real hardware. Harness must be quick to launch and tear down. +- **Subjective voice quality**: Objective metrics + a small set of "golden" household-style prompts for human listening tests. +- **NIM model availability & download time**: Pre-pull images; document exact tags used in every report. +- **Reproducibility**: Pin exact image digests + compose files + env in every report. +- **Measurement overhead**: Keep the harness itself as lightweight as possible so it doesn't distort the numbers. + +## 7. Deliverables + +- Benchmark harness + reporting tooling (reusable for future experiments). +- 4–6 committed benchmark reports with clear deltas. +- Updated default deployment configuration for the chosen sweet spot. +- Living documentation (in `specs/007-.../` and `docs/`) that future team members (or future us) can follow. +- Clear go/no-go + resource numbers for moving to 2× DGX Spark. + +## 8. Timeline Philosophy + +We are not optimizing in the dark. Every day of work on this branch should produce either: +- A new measurement, or +- A concrete code/config change that is immediately measured against the previous baseline. + +This keeps the loop tight and the excitement high. + +--- + +**Ready to cut.** Once the spec PR is reviewed/merged, we will land the first pieces of the harness and capture the all-important baseline numbers on the actual DGX Spark hardware. \ No newline at end of file diff --git a/specs/007-dgx-hardware-optimization/spec.md b/specs/007-dgx-hardware-optimization/spec.md new file mode 100644 index 0000000..c140f9c --- /dev/null +++ b/specs/007-dgx-hardware-optimization/spec.md @@ -0,0 +1,220 @@ +# Spec: DGX Spark Hardware Optimization & Sweet-Spot Strategy + +**Feature ID**: 007-dgx-hardware-optimization +**Status**: Draft / Strategy & Benchmarking Spec +**Related to**: 001-voice-dgx-spark-agent, 002-multi-user-support, 003-deployment-infrastructure +**Created**: 2025-05-22 +**Branch**: `feat/007-dgx-hardware-optimization` + +## Overview + +We have reached the point where we need a deliberate, measurable optimization strategy for the voice-first LangGraph + Supermemory agent running on NVIDIA DGX Spark hardware (single node today, with an eye toward 2× DGX Spark). + +Current production configuration (as of Feature 001): +- LLM: `nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b:latest` (120B hybrid MoE/Mamba, ~12B active params, 1M native context) +- Voice: Full NVIDIA Riva (NeMo ASR + TTS) via gRPC sidecar +- Agent: LangGraph StateGraph with proactive Supermemory injection + reactive memory tools +- Deployment: Docker on DGX Spark (single node) + +The user wants: +- Formal expectations so we can **benchmark expected vs. actual** performance. +- Clear headroom calculations for Riva + Nemotron (current and future models). +- Evaluation of **dropping Riva entirely** for English-only household use (rely on lighter Nemotron-era audio paths or dedicated small speech models). +- Resource profile for the "big boi" Nemotron-4 340B-class model and what headroom remains for the rest of the application. +- Expected gains and architecture implications of a **2× DGX Spark** multi-node setup. +- Trade-off framework and a process for finding the **sweet spot** under real hardware constraints. + +This spec establishes the measurement baseline, decision framework, and optimization levers. Implementation and concrete experiments will follow in subsequent plan/tasks once this spec is reviewed and locked. + +## Hardware Reality: DGX Spark (Single Node) + +Key characteristics that drive every optimization decision: + +- **GB10 Grace Blackwell Superchip** (Blackwell GPU + 20-core Arm CPU: 10 performance + 10 efficiency cores) +- **128 GB unified LPDDR5X memory** (coherent between CPU and GPU, ~273 GB/s bandwidth). This is the single most important constraint: model weights, KV cache, activations, Riva models, agent process, OS, Docker overhead, and audio I/O all compete for the same pool. There is no separate "VRAM." +- High AI throughput (up to ~1 PFLOP FP4 on Tensor Cores) but memory-bound for large models + long context. +- Storage: 4 TB NVMe +- Networking: 10 GbE + dual 100/200 GbE ConnectX-7 (RDMA capable) +- Power/thermals: ~140 W SoC, ~240 W PSU, designed for quiet/home-lab operation + +**Implication**: Every added service (Riva, larger model, longer context, multiple concurrent family conversations) directly reduces headroom for the "rest of the app" (LangGraph execution, Supermemory client calls, VAD, playback, future tools). + +## Current Baseline (What We Are Running Today) + +- **LLM**: nemotron-3-super-120b-a12b (120B total / ~12–12.7B active per token via hybrid MoE + Mamba). Excellent agentic/tool-calling and long-context reasoning — ideal for our Supermemory injection + multi-turn household conversations. +- **Context**: Native 1M tokens (practical NIM limits often 128K–256K depending on profile and KV precision). +- **Voice**: Full Riva stack (Parakeet-class ASR + high-quality TTS, multilingual capable). +- **Expected characteristics** (to be validated on hardware): + - Model load + idle memory: Significant fraction of 128 GB (exact TBD via NIM profile). + - Real-time voice turn latency (end-of-speech → first audio out): Target sub-second natural feel. + - Concurrent family users: Currently designed for single primary user; multi-user will increase memory pressure. + +We do **not** yet have hard numbers on this exact DGX Spark + Docker + Riva + 120B combination. This spec exists to create those numbers systematically. + +## Model Comparison + +| Model | Params (Total / Active) | Context | Architecture | Expected Footprint (Single Spark) | Strengths for Our Use Case | Weaknesses / Risks | Voice Latency Impact | +|-------|--------------------------|---------|--------------|-----------------------------------|----------------------------|--------------------|----------------------| +| **nemotron-3-super-120b-a12b** (current) | 120B / ~12B active | 1M native (NIM ~128–256K practical) | Hybrid Mamba + Transformer MoE | High (but runnable per community reports; tight with Riva + long ctx) | Best agentic reasoning, tool use, long-horizon memory recall, retains large Supermemory context without constant re-fetch | Highest memory/latency of the three practical options; risk of swapping under load | Higher TTFT + decode latency vs lighter models | +| **llama-3.3-nemotron-super-49b-v1.5** (strong candidate) | 49B dense | 128K | NAS-optimized dense Transformer | Medium (comfortable headroom on single Spark) | Excellent accuracy/efficiency; fast tokens/s; proven on H100-class; lower latency, more room for Riva or concurrent users | Smaller context than 120B (may require more aggressive summarization) | Best-in-class for its size; fastest turn times of the three | +| **nemotron-4-340b-instruct** ("big boi") | 340B dense | 4K native (extendable) | Dense Transformer | Impractical on single node even heavily quantized; feasible on 2× via tensor-parallel | Maximum raw intelligence and instruction following; potential "reasoning brain" for hardest queries | Enormous memory (hundreds of GB raw); high latency even sharded; overkill for most voice turns | Significantly higher latency; best used selectively or for offline tasks | + +**Recommendation for primary inference path**: Start with the 49B v1.5 as the default "daily driver" for voice responsiveness while keeping the 120B as an optional "deep thinker" that can be swapped in for complex multi-step planning or heavy memory synthesis. + +## Audio Stack: Can We Axe Riva? + +Current: Full Riva (enterprise-grade, multi-language, multiple models for ASR + TTS). + +For an **English-only household** (Derek + family), the multilingual enterprise features are mostly wasted. + +**Lighter English-only alternatives**: +- Pin to **Parakeet 1.1B CTC English** (or smaller 0.6B variants) via dedicated lightweight ASR NIM or direct NeMo inference — typically 2–8 GB for real-time streaming. +- High-quality **English-only TTS** (single or a few voices) — another 4–8 GB. +- Total audio stack: **4–12 GB** instead of 10–25+ GB for full Riva. + +**Benefits of dropping full Riva**: +- Reclaim 8–15+ GB of unified memory → directly usable for larger KV cache (longer effective context), higher quality model, or concurrent family sessions. +- Simpler deployment (fewer sidecars, smaller attack surface, faster startup). +- Lower CPU/GPU contention during voice turns. + +**Risks / Trade-offs**: +- Lose easy future multilingual support (acceptable per current requirement). +- Must validate English quality and latency of the lighter path (Parakeet CTC is already very strong for English). +- Potential future desire for "voice cloning" or multiple family voices — still doable with lighter dedicated voices. + +**Conclusion**: Yes — for the English-only family voice agent we can (and probably should) replace full Riva with a minimal English Parakeet + English TTS profile (or emerging smaller NVIDIA speech NIMs). This is one of the highest-leverage single changes available today. + +## Headroom Analysis (Single DGX Spark, 128 GB Unified) + +Rough engineering estimates (to be replaced by measured data): + +**Always-present baseline**: +- OS + Docker + non-root agent container + Python + sounddevice + VAD + Supermemory client + LangGraph overhead + checkpointers: **8–15 GB** + +**LLM (weights + typical KV for voice turns)**: +- 120B MoE (optimized NIM FP8/lower): **40–70 GB** depending on active context window and quantization profile. (Active ~12B helps enormously vs dense 120B.) +- 49B dense (optimized): **25–45 GB** — significantly more comfortable. +- 340B (even heavily quantized): **150+ GB** — impossible on single node without extreme measures. + +**Audio (Riva vs lighter)**: +- Full Riva concurrent ASR+TTS: **10–25 GB** +- English-only Parakeet + TTS: **4–12 GB** + +**Headroom for "the rest of the app"** (reactive tool calls, memory injection, future vision/tools, burst concurrency): +- Current 120B + full Riva: **Very tight** (often <10–15 GB free under load). Risk of OOM, swapping, or forced context truncation during long family conversations. +- 49B + lighter audio: **Healthy headroom** (20–40+ GB free) → room for 2–4 concurrent family members, longer context windows, or future capabilities. +- 120B + lighter audio: **Recoverable** — may be the pragmatic sweet spot for intelligence + responsiveness. + +**Key insight**: The biggest single lever for headroom today is **replacing full Riva with an English-only lightweight audio path**. The second biggest is **model choice** (49B vs 120B MoE). + +## Multi-Node (2× DGX Spark) Projections + +Two nodes give us: +- 256 GB total unified memory (128 GB each) +- Excellent inter-node bandwidth via dual 200 GbE ConnectX-7 RDMA (theoretical very high; early community reports ~8 GB/s bidirectional practical) +- Official 2-node support from NVIDIA; community has run 3+ nodes + +**What we gain**: +1. **Run the 340B "big boi"** via tensor-parallel / pipeline-parallel sharding across the two nodes. Each node holds ~half the weights + portion of KV. Feasible but higher latency than the 49B/120B on single node. Best used as an on-demand "deep reasoning" service rather than the primary voice responder. +2. **Scale the current 120B** with much larger effective context (or higher batch / concurrent sessions) without swapping. +3. **Separate concerns**: Node A = heavy LLM inference (120B or 340B shard); Node B = voice I/O + lighter agent orchestration + memory tools. Reduces contention on the voice path. +4. **Family concurrency**: Comfortably support 4–8+ simultaneous or overlapping household conversations. +5. **Redundancy / failover** for an always-on home device. +6. **Future headroom**: Add vision models, additional tools, or a second "specialist" LLM without immediate hardware purchase. + +**What it costs**: +- Complexity: multi-node Docker Compose / Kubernetes or custom orchestration, RDMA networking setup, model sharding configuration. +- Latency: Cross-node communication for tensor-parallel adds some overhead (acceptable for the 340B case; less ideal for every voice turn). +- Power/heat/noise: Two units running. +- Cost and physical space. + +**Expectation**: 2× setup moves us from "tight single-node optimization" to "comfortable production with room to grow." It is the natural next hardware step once we outgrow a well-tuned single Spark. + +## Benchmarking Methodology & Expectations vs. Actuals + +We will establish a repeatable benchmark harness before making major changes. + +### Core Metrics (voice turn focus) +- **Voice turn latency**: Time from end-of-speech (VAD) to first audio chunk out (p50 / p95). Target: <800 ms feels natural; <1.2 s acceptable. +- **TTFT** (time to first token) + **tokens per second** during generation. +- **End-to-end perceived latency** including memory injection + tool use. +- **Memory usage at key points**: Idle after load, during active voice turn, peak during long-context recall + multiple tool calls. +- **Concurrency**: Max stable simultaneous family members before degradation. +- **Stability**: No OOM / swap over 30–60 min household usage sessions. +- **Thermals / power / noise**: Important for a living-room/home device. + +### Baseline Capture (First Experiment on Current Stack) +1. Single DGX Spark, current 120B + full Riva Docker Compose. +2. Clean boot, measure idle memory. +3. Run scripted voice sessions (single user, then 2–3 overlapping). +4. Capture all metrics above + full `nvidia-smi` / container memory + system logs. +5. Document exact NIM profiles, quantization settings, Riva config, and context management strategy used. + +### Subsequent Experiments (Compare Against Baseline) +- 120B + lighter English audio only +- 49B v1.5 + lighter English audio +- Same with aggressive memory summarization / context window tuning +- 2× node configurations (once hardware available) + +Every change must be accompanied by before/after numbers against the baseline. "It feels faster" is not enough — we log hard data. + +## Optimization Levers & Trade-off Framework + +Primary levers (ranked by expected impact on single-node headroom + latency): + +1. **Audio stack reduction** (full Riva → English Parakeet + TTS): Highest immediate win. +2. **Model swap** (120B MoE → 49B dense): Large win on latency and headroom; acceptable quality trade for most turns. +3. **Context strategy** (aggressive summarization + proactive injection vs. raw long context): Reduces KV pressure and improves recall quality. +4. **Quantization / NIM profile tuning**: FP8, NVFP4, lower KV precision where quality allows. +5. **Concurrency limits & backpressure**: Limit parallel family sessions or queue intelligently. +6. **Process placement** (future): Move audio to a dedicated lightweight container or even separate node. +7. **2× node scaling**: When single-node sweet spot is exhausted. + +### Decision Matrix (Example) + +| Configuration | Expected Headroom | Expected Voice Latency | Intelligence Level | Multi-User Comfort | Recommendation | +|---------------|-------------------|------------------------|--------------------|--------------------|----------------| +| 120B + Full Riva | Low | Medium-High | Highest | Poor | Baseline only; optimize away | +| 120B + Light Audio | Medium | Medium | Highest | Good | Strong candidate if quality holds | +| 49B + Light Audio | High | Lowest | Very High | Excellent | Default daily driver target | +| 340B (2× sharded) | N/A (multi-node) | High | Maximum | Excellent | On-demand specialist brain | + +## Recommended Path to Sweet Spot (Single Node First) + +1. **Immediate (this branch / next sprint)**: Capture rigorous baseline on current 120B + Riva. +2. **High-leverage experiment**: Replace Riva with minimal English audio stack; re-benchmark. +3. **Model A/B**: Stand up 49B v1.5 side-by-side; measure latency + memory + subjective quality on household-style prompts + memory recall tasks. +4. **Context tuning**: Implement or improve summarization + injection strategy; measure impact on effective memory quality vs. KV usage. +5. **Decision gate**: Choose primary model + audio stack for the family deployment based on data. +6. **2× node phase**: Once single-node sweet spot is locked and we need more (concurrency, 340B, or future capabilities), move to multi-node architecture. + +## Success Criteria + +- We have a documented, reproducible benchmark baseline for the current stack. +- We have measured data (not guesses) for at least two alternative configurations (light audio, 49B model). +- We can articulate "the sweet spot" with numbers: model choice, audio stack, max comfortable context, max concurrent users, and expected voice turn latency. +- The chosen configuration leaves **measurable, comfortable headroom** (>15–20 GB) for the agent, memory system, and future features under typical household load. +- We have a clear, data-driven recommendation on whether/when to move to 2× DGX Spark and what that unlocks. + +## Open Questions & Risks + +- Exact real-world memory footprint of the 120B NIM on DGX Spark unified memory (with our Docker setup) — highest priority unknown. +- Quality delta between full Riva voices and lighter English TTS options for family members (subjective but important). +- Whether 128K context on the 49B is "enough" given our Supermemory + summarization strategy, or whether we will miss the 1M capability. +- Practical limits of 2-node RDMA tensor-parallel for the 340B in a home setting (latency, stability, complexity). +- Future desire for on-device voice cloning or multiple distinct family voices — does this push us back toward a heavier audio stack later? +- Energy / heat / noise profile of sustained operation (especially 2×) in a living space. + +## Next Steps + +1. Review and approve this spec (user + team). +2. Create `plan.md` and `tasks.md` under `specs/007-dgx-hardware-optimization/` following the established SDD process. +3. Implement the benchmark harness + first baseline capture. +4. Run the high-leverage experiments (audio reduction, model comparison). +5. Iterate to the sweet spot with data in hand. + +--- + +**We are getting there.** This spec gives us the map, the measuring stick, and the decision framework so we can move from "it works" to "it is optimal under our real constraints" with confidence and excitement. + +**Status**: Ready for review. Once approved, we will commit the spec and proceed to planning the concrete benchmarking and optimization work. \ No newline at end of file diff --git a/specs/007-dgx-hardware-optimization/tasks.md b/specs/007-dgx-hardware-optimization/tasks.md new file mode 100644 index 0000000..a20bf90 --- /dev/null +++ b/specs/007-dgx-hardware-optimization/tasks.md @@ -0,0 +1,144 @@ +# Tasks: DGX Spark Hardware Optimization & Sweet-Spot Discovery (007) + +**Feature**: 007-dgx-hardware-optimization +**Related Spec**: [spec.md](./spec.md) +**Related Plan**: [plan.md](./plan.md) +**Status**: Ready for implementation +**Branch**: `feat/007-dgx-hardware-optimization-impl` + +This document breaks the work into small, dependency-ordered, checkable tasks. The first priority is always **capturing a trustworthy baseline** before any changes. + +Mark tasks complete only after the work is committed and (where applicable) the corresponding benchmark report is added. + +--- + +## Phase 0 – Baseline Capture (Highest Priority) + +### T0.1 – Project Structure for Benchmarking +- [ ] Create `benchmarks/` directory at repo root with `__init__.py` and `README.md` explaining the harness. +- [ ] Add `benchmarks/reports/` (gitignored except for `.gitkeep` and example reports). +- [ ] Update `.gitignore` if needed for report artifacts. + +### T0.2 – Benchmark Runner Skeleton +- [ ] Create `benchmarks/runner.py` (or `benchmarks/cli.py`) using Typer or argparse for a `thelab-bench` entry point. +- [ ] Support basic flags: `--scenario short|long|concurrent`, `--duration`, `--user`, `--output-dir`. +- [ ] Implement structured JSON logging of events (turn start, end-of-speech, transcript, agent response, first audio, errors). + +### T0.3 – Instrumentation in Voice Layer +- [ ] Add optional timing / event hooks in `src/thelab_langchain/voice/orchestrator.py` (or a small `timing.py` helper). + - Record: `end_of_speech_ts`, `transcript_ready_ts`, `agent_first_token_ts`, `first_audio_out_ts`. +- [ ] Make instrumentation toggleable via env var (`BENCHMARK_MODE=1`) so it doesn't affect normal runs. +- [ ] Ensure the existing voice loop still works cleanly when the hooks are disabled. + +### T0.4 – Memory & System Metrics Collection +- [ ] Add a lightweight sampler (background thread or separate process) that records: + - Docker container memory / CPU (via `docker stats` API or subprocess). + - `nvidia-smi` output (memory, utilization, power). + - System RAM / swap. +- [ ] Integrate sampling into the benchmark runner for the duration of a run. + +### T0.5 – Baseline Run on DGX (Current Stack) +- [ ] On clean DGX Spark, pull the exact current images (120B + full Riva). +- [ ] Run the benchmark harness against the existing `docker-compose.yml`. +- [ ] Execute at least one "short turns" session and one "long household conversation" session. +- [ ] Capture full report (metrics, logs, nvidia-smi samples, container stats). +- [ ] Commit the report as `benchmarks/reports/2025-05-XX-baseline-120b-riva/` (with `summary.md` + `raw/`). + +### T0.6 – Baseline Documentation +- [ ] Write `specs/007-dgx-hardware-optimization/results/phase-0-baseline.md` summarizing the measured numbers against the expectations in the spec. +- [ ] Update the decision matrix in the spec (or a living `decision-log.md`) with actual data. + +--- + +## Phase 1 – Audio Stack Reduction + +### T1.1 – Lightweight English Audio Service Definition +- [ ] Research and select the exact lighter image(s): Parakeet English CTC NIM (or equivalent small ASR) + English TTS. +- [ ] Create `docker-compose.light-audio.yml` (or profile) that replaces the full Riva service with the slimmed version. +- [ ] Document exact tags, ports, and healthcheck expectations in the compose file and a small `audio-profiles.md`. + +### T1.2 – Agent / Orchestrator Compatibility +- [ ] Verify (or lightly adapt) the Riva gRPC client code to work with the new lighter service (same gRPC surface if possible). +- [ ] Add `AUDIO_PROFILE=light` (or `full`) env handling in config / compose. + +### T1.3 – Light Audio Benchmark Run +- [ ] Deploy the light-audio profile on DGX. +- [ ] Re-run the same benchmark scenarios used in baseline. +- [ ] Produce report `benchmarks/reports/...-light-audio/`. + +### T1.4 – Phase 1 Gate & Decision +- [ ] Compare memory headroom, voice turn latency (p50/p95), and subjective quality. +- [ ] Write `results/phase-1-audio-reduction.md`. +- [ ] Decision recorded: adopt light audio as new default (or keep full Riva). + +--- + +## Phase 2 – Model Comparison (49B Candidate) + +### T2.1 – Second LLM Service in Compose +- [ ] Add support for `llama-3.3-nemotron-super-49b-v1.5` (new service definition or override). +- [ ] Make the agent LLM endpoint configurable (`LLM_BASE_URL`, `LLM_MODEL` or similar) so we can point at different NIMs without code changes. +- [ ] Create `docker-compose.49b.yml` profile. + +### T2.2 – 49B Benchmark Runs +- [ ] With the winning audio stack from Phase 1, run identical scenarios on the 49B model. +- [ ] Capture full metrics + at least one side-by-side human listening session for quality. +- [ ] Produce report and `results/phase-2-49b-comparison.md`. + +### T2.3 – Phase 2 Gate +- [ ] Update decision matrix with real latency + headroom numbers. +- [ ] Lock primary model recommendation (49B daily driver + 120B optional deep mode is the current hypothesis). + +--- + +## Phase 3 – Context & Efficiency Tuning + +### T3.1 – Summarization / Context Window Experiments +- [ ] Enhance the memory injection logic (or add a summarizer node) to keep effective context while reducing KV cache pressure. +- [ ] Define 2–3 different context strategies as config options. +- [ ] Benchmark the impact on memory usage, recall quality (test prompts that rely on older memories), and latency. + +### T3.2 – Concurrency / Multi-User Stability +- [ ] Add a concurrent benchmark mode that simulates 2–4 overlapping household conversations. +- [ ] Measure stability and headroom under load with the chosen model + audio. +- [ ] Document maximum comfortable concurrent users. + +--- + +## Phase 4 – Sweet-Spot Operationalization + +### T4.1 – Default Configuration Update +- [ ] Update the main `docker-compose.yml` (or make the winning profiles the easy defaults via env). +- [ ] Add high-level `make` targets: `make benchmark`, `make profile-sweet-spot`, etc. +- [ ] Update `docs/development.md` and any DGX runbooks with the new recommended command sequence. + +### T4.2 – Final Results Package +- [ ] Write `results/final-sweet-spot.md` with the locked configuration, all key metrics, and rationale. +- [ ] Update the top-level spec with "Measured Sweet Spot" section (post-experiment). + +### T4.3 – Cleanup & Polish +- [ ] Remove or clearly mark temporary instrumentation so normal runs have zero overhead. +- [ ] Ensure the benchmark harness is reusable and well-documented. + +--- + +## Phase 5 – 2× DGX Spark Preparation (Stretch / Future) + +- [ ] Document multi-node networking requirements (ConnectX-7 RDMA setup on the two Sparks). +- [ ] Create initial `docker-compose.2node.yml` skeleton or separate stack definitions. +- [ ] Optional spike: basic tensor-parallel or service-separation test on 2 nodes (if hardware available). +- [ ] Update the 007 spec with concrete multi-node headroom and latency expectations based on real data. + +--- + +## Cross-Cutting / Ongoing + +- [ ] Keep all benchmark reports committed with pinned image digests and exact compose/env used. +- [ ] Every significant change on this branch must be accompanied by a new report or clear "no measurement impact" note. +- [ ] Update the project tracker with the current phase as we progress. + +--- + +**First actionable tasks**: T0.1 – T0.3 (get the harness skeleton + instrumentation in place) so that the very first DGX run (T0.5) produces trustworthy, comparable numbers. + +Let's go get those baseline numbers! \ No newline at end of file diff --git a/src/thelab_langchain/__init__.py b/src/thelab_langchain/__init__.py new file mode 100644 index 0000000..372c199 --- /dev/null +++ b/src/thelab_langchain/__init__.py @@ -0,0 +1,28 @@ +"""thelab-langchain: LangChain + Supermemory integration with Grok (xAI) or Anthropic. + +Includes a voice layer for NVIDIA NeMo / Riva on DGX Spark. +""" + +from .agent.graph import get_agent +from .chat import MemoryChat, MemoryContext +from .config import Settings, settings + + +# Voice components are optional (require audio runtime libs like libportaudio2). +# They are imported on demand via `from thelab_langchain.voice import ...` +# or when the `thelab-chat voice` subcommand is used. +def __getattr__(name: str): + if name == "VoiceOrchestrator": + from .voice import VoiceOrchestrator as _VoiceOrchestrator + return _VoiceOrchestrator + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + +__all__ = [ + "MemoryChat", + "MemoryContext", + "Settings", + "settings", + "VoiceOrchestrator", + "get_agent", +] +__version__ = "0.1.0" diff --git a/src/thelab_langchain/agent/__init__.py b/src/thelab_langchain/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/thelab_langchain/agent/graph.py b/src/thelab_langchain/agent/graph.py new file mode 100644 index 0000000..fba635e --- /dev/null +++ b/src/thelab_langchain/agent/graph.py @@ -0,0 +1,137 @@ +""" +LangGraph agent definition for the TheLab voice agent. + +This module is responsible for constructing the reasoning brain +that sits behind the voice interface. +""" + +from __future__ import annotations + +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage +from langgraph.graph import END, StateGraph +from langgraph.prebuilt import ToolNode + +from ..llm import get_chat_model +from .state import AgentState +from .tools.memory import create_memory_tools + + +def _memory_injection(state: AgentState) -> dict: + """ + Proactively pull relevant long-term memory from Supermemory before the LLM thinks. + + This gives the agent immediate context about the user without requiring the + model to decide to call tools on every turn. + """ + user_id = getattr(state, "user_id", "default-user") + + # Get the last user utterance to use as a good recall query + last_user_msg = "" + for msg in reversed(state.messages): + if isinstance(msg, HumanMessage): + last_user_msg = getattr(msg, "content", str(msg)) + break + + tools = create_memory_tools(user_id) + + # Find the tools by name + profile_tool = next((t for t in tools if t.name == "get_user_profile"), None) + recall_tool = next((t for t in tools if t.name == "recall_memories"), None) + + profile_context = "" + if profile_tool: + try: + profile_context = profile_tool.invoke({"query": last_user_msg or None}) + except Exception: + profile_context = "" + + memory_context = "" + if recall_tool and last_user_msg: + try: + memory_context = recall_tool.invoke({"query": last_user_msg, "limit": 3}) + except Exception: + memory_context = "" + + combined = "" + if profile_context: + combined += profile_context + "\n\n" + if memory_context: + combined += f"## Relevant Long-term Memories\n{memory_context}" + + if not combined: + return {} + + # Inject raw context directly — the main LLM handles it fine and this avoids + # an extra LLM round-trip that adds 500-1000ms of latency per voice turn. + injection = SystemMessage(content=f"## User Context (from long-term memory)\n{combined.strip()}") + + # Prepend the injection so it's early context + return {"messages": [injection] + list(state.messages)} + + +def _call_llm(state: AgentState, tools: list) -> dict: + """Call the LLM (with tools bound) on the current messages.""" + llm = get_chat_model() + llm_with_tools = llm.bind_tools(tools) + response = llm_with_tools.invoke(state.messages) + return {"messages": [response]} + + +def _execute_tools(state: AgentState, tools: list) -> dict: + """Execute any tool calls requested by the LLM.""" + tool_node = ToolNode(tools) + result = tool_node.invoke(state) + return result + + +def _should_continue(state: AgentState) -> str: + """Decide whether to continue to tools or end.""" + last_message = state.messages[-1] + if isinstance(last_message, AIMessage) and getattr(last_message, "tool_calls", None): + return "execute_tools" + return END + + +def build_agent_graph(user_id: str = "default-user") -> StateGraph: + """ + Build the main LangGraph for the voice agent. + + Architecture: + - memory_injection (proactive Supermemory context) + - call_llm (with Supermemory tools bound) + - execute_tools (if the model requested any) + - loop back to call_llm if tools were used + + This enables both proactive memory injection and reactive tool use + (the model can now decide to call store_memory, recall_memories, etc.). + """ + tools = create_memory_tools(user_id) + + workflow = StateGraph(AgentState) + + # Bind tools into the nodes via closures + def call_llm(state: AgentState): + return _call_llm(state, tools) + + def execute_tools(state: AgentState): + return _execute_tools(state, tools) + + workflow.add_node("memory_injection", _memory_injection) + workflow.add_node("call_llm", call_llm) + workflow.add_node("execute_tools", execute_tools) + + workflow.set_entry_point("memory_injection") + workflow.add_edge("memory_injection", "call_llm") + workflow.add_conditional_edges("call_llm", _should_continue, { + "execute_tools": "execute_tools", + END: END, + }) + workflow.add_edge("execute_tools", "call_llm") + + return workflow + + +def get_agent(user_id: str = "default-user"): + """Returns a compiled runnable LangGraph agent for the given user.""" + graph = build_agent_graph(user_id=user_id) + return graph.compile() diff --git a/src/thelab_langchain/agent/state.py b/src/thelab_langchain/agent/state.py new file mode 100644 index 0000000..3ce0418 --- /dev/null +++ b/src/thelab_langchain/agent/state.py @@ -0,0 +1,32 @@ +""" +Agent state definition for the LangGraph-powered Supermemory agent. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Annotated, Any + +from langchain_core.messages import BaseMessage +from langgraph.graph.message import add_messages +from pydantic import BaseModel, Field + + +class AgentState(BaseModel): + """The state passed between nodes in the LangGraph agent.""" + + # Conversation history (automatically managed by LangGraph with add_messages reducer) + messages: Annotated[Sequence[BaseMessage], add_messages] = Field(default_factory=list) + + # Identity & session + user_id: str = Field(..., description="Supermemory container_tag / user namespace") + thread_id: str = Field(default="default", description="Persistent session identifier") + + # Long-term memory context (injected by memory nodes) + long_term_context: str = Field(default="", description="Relevant memories + profile injected into the prompt") + + # Scratchpad for tools / intermediate results + scratchpad: dict[str, Any] = Field(default_factory=dict) + + # Control flags + needs_memory_refresh: bool = Field(default=False) diff --git a/src/thelab_langchain/agent/tools/__init__.py b/src/thelab_langchain/agent/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/thelab_langchain/agent/tools/memory.py b/src/thelab_langchain/agent/tools/memory.py new file mode 100644 index 0000000..e36bfab --- /dev/null +++ b/src/thelab_langchain/agent/tools/memory.py @@ -0,0 +1,108 @@ +""" +Supermemory operations exposed as LangChain @tool functions. + +These become the agent's interface to long-term user memory. +The agent can choose when to recall or store information. + +Tools are created via `create_memory_tools(user_id)` so they are properly +scoped per user (critical for future multi-user support). +""" + +from __future__ import annotations + +import os +from typing import Any + +from langchain_core.tools import BaseTool, tool +from pydantic import BaseModel, Field +from supermemory import Supermemory + + +def _get_memory_client() -> Supermemory: + """Lazy singleton Supermemory client.""" + api_key = os.getenv("SUPERMEMORY_API_KEY") + if not api_key: + raise RuntimeError("SUPERMEMORY_API_KEY must be set in environment") + return Supermemory(api_key=api_key) + + +class RecallMemoriesInput(BaseModel): + query: str = Field(..., description="Semantic search query to find relevant past memories") + limit: int = Field(default=5, description="Maximum number of memories to return") + + +class StoreMemoryInput(BaseModel): + content: str = Field(..., description="The memory content to store permanently") + metadata: dict[str, Any] | None = Field( + default=None, description="Optional structured metadata (tags, project, type, etc.)" + ) + + +class GetProfileInput(BaseModel): + query: str | None = Field( + default=None, + description="Optional query to also pull relevant memories alongside the profile", + ) + + +def create_memory_tools(user_id: str) -> list[BaseTool]: + """ + Factory that returns Supermemory tools bound to a specific user. + + This is the recommended way to create the tools so they are + correctly isolated per user (supports multi-user households). + """ + client = _get_memory_client() + + @tool("recall_memories", args_schema=RecallMemoriesInput) + def recall_memories(query: str, limit: int = 5) -> str: + """Search the user's long-term memory for relevant past conversations, facts, and context.""" + results = client.search.memories( + q=query, + container_tag=user_id, + limit=limit, + ) + if not results or not getattr(results, "results", None): + return "No relevant memories found for this query." + + formatted = [] + for r in results.results: + text = getattr(r, "memory", None) or getattr(r, "chunk", None) or str(r) + if text: + formatted.append(f"- {text}") + return "\n".join(formatted[:limit]) + + @tool("store_memory", args_schema=StoreMemoryInput) + def store_memory(content: str, metadata: dict[str, Any] | None = None) -> str: + """Store important information about the user into long-term memory.""" + client.add( + content=content, + container_tag=user_id, + metadata=metadata or {}, + ) + return f"Successfully stored memory: {content[:100]}..." + + @tool("get_user_profile", args_schema=GetProfileInput) + def get_user_profile(query: str | None = None) -> str: + """Retrieve the user's automatically generated long-term profile and relevant memories.""" + result = client.profile( + container_tag=user_id, + q=query or "current context and preferences", + ) + profile = getattr(result, "profile", None) or {} + static = getattr(profile, "static", None) or [] + dynamic = getattr(profile, "dynamic", None) or [] + + lines = ["## User Profile"] + if static: + lines.append("\n### Long-term Facts") + lines.extend(f"- {s}" for s in static) + if dynamic: + lines.append("\n### Recent Activity & Focus") + lines.extend(f"- {d}" for d in dynamic) + + if len(lines) == 1: + return "No profile information available yet for this user." + return "\n".join(lines) + + return [recall_memories, store_memory, get_user_profile] diff --git a/src/thelab_langchain/chat.py b/src/thelab_langchain/chat.py new file mode 100644 index 0000000..3b3f564 --- /dev/null +++ b/src/thelab_langchain/chat.py @@ -0,0 +1,181 @@ +"""Memory-aware chat using LangChain (Grok/Anthropic) + Supermemory.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage +from rich.console import Console +from rich.panel import Panel +from supermemory import Supermemory + +from .config import settings +from .llm import get_chat_model + +console = Console() + + +@dataclass +class MemoryContext: + """Structured context returned from Supermemory profile + search.""" + + static_facts: list[str] = field(default_factory=list) + dynamic_context: list[str] = field(default_factory=list) + relevant_memories: list[str] = field(default_factory=list) + raw: Any = None # original Supermemory result for advanced use + + def to_prompt_block(self) -> str: + """Format as a readable context block for the system prompt.""" + lines: list[str] = ["## User Memory Context"] + + if self.static_facts: + lines.append("\n### Long-term Profile") + lines.extend(f"- {fact}" for fact in self.static_facts) + + if self.dynamic_context: + lines.append("\n### Recent Activity") + lines.extend(f"- {ctx}" for ctx in self.dynamic_context) + + if self.relevant_memories: + lines.append("\n### Relevant Past Memories") + lines.extend(f"- {mem}" for mem in self.relevant_memories[:6]) + + if len(lines) == 1: + lines.append("No prior memories or profile for this user yet.") + + return "\n".join(lines) + + +class MemoryChat: + """Conversational agent that uses Supermemory for long-term recall and LangChain for reasoning.""" + + def __init__(self, user_id: str | None = None) -> None: + settings.validate_keys() + + self.user_id = user_id or settings.default_user_id + self.memory = Supermemory(api_key=settings.supermemory_api_key.get_secret_value()) + + # Initialize the chosen LLM using the central factory (supports xai, anthropic, openai_compatible) + self.llm = get_chat_model() + + if settings.llm_provider == "xai": + self.provider_name = "Grok (xAI)" + elif settings.llm_provider == "anthropic": + self.provider_name = "Claude (Anthropic)" + else: + self.provider_name = f"Local ({settings.llm_model})" + + self.conversation: list[HumanMessage | AIMessage] = [] + + def _get_memory_context(self, query: str) -> MemoryContext: + """Fetch profile + semantically relevant memories for the current query.""" + try: + result = self.memory.profile( + container_tag=self.user_id, + q=query, + threshold=0.55, + ) + except Exception as exc: # Supermemory errors are usually network / auth + console.print(f"[red]Supermemory error:[/red] {exc}") + return MemoryContext() + + profile = getattr(result, "profile", None) or {} + search_results = getattr(result, "search_results", None) + + static = getattr(profile, "static", None) or [] + dynamic = getattr(profile, "dynamic", None) or [] + + memories: list[str] = [] + if search_results and getattr(search_results, "results", None): + for r in search_results.results: + text = getattr(r, "memory", None) or getattr(r, "chunk", None) or str(r) + if text: + memories.append(text) + + return MemoryContext( + static_facts=static, + dynamic_context=dynamic, + relevant_memories=memories, + raw=result, + ) + + def _build_prompt(self, user_message: str, memory_ctx: MemoryContext) -> list[SystemMessage | HumanMessage | AIMessage]: + """Construct the full message list for the LLM including memory context.""" + system_content = f"""You are a helpful, thoughtful assistant with excellent long-term memory. + +You are powered by {self.provider_name} and backed by Supermemory for persistent user context. + +{memory_ctx.to_prompt_block()} + +Instructions: +- Use the memory context above to personalize your answers. +- Reference past facts, preferences, or conversations naturally when relevant. +- If the user is new, be friendly and help them build their profile by asking light questions. +- Never mention the internal "memory context" or Supermemory directly unless the user asks. +- Be concise but warm. +""" + + messages: list[SystemMessage | HumanMessage | AIMessage] = [ + SystemMessage(content=system_content.strip()) + ] + messages.extend(self.conversation[-10:]) # keep recent turns for coherence + messages.append(HumanMessage(content=user_message)) + return messages + + def chat(self, message: str) -> str: + """Process a user message, recall memory, call LLM, and persist the turn.""" + console.print(f"\n[dim]Fetching memory for user '{self.user_id}'...[/dim]") + memory_ctx = self._get_memory_context(message) + + # Show a tiny summary of what memory we got (for demo transparency) + if memory_ctx.static_facts or memory_ctx.dynamic_context or memory_ctx.relevant_memories: + summary = ( + f"[green]✓[/green] Profile: {len(memory_ctx.static_facts)} static, " + f"{len(memory_ctx.dynamic_context)} dynamic, " + f"{len(memory_ctx.relevant_memories)} relevant memories pulled." + ) + console.print(summary) + + prompt_messages = self._build_prompt(message, memory_ctx) + + console.print(f"[dim]Calling {self.provider_name} ({settings.llm_model})...[/dim]") + response: AIMessage = self.llm.invoke(prompt_messages) # type: ignore[assignment] + + # Persist to long-term memory + try: + self.memory.add( + content=f"User: {message}\nAssistant: {response.content}", + container_tag=self.user_id, + metadata={ + "provider": settings.llm_provider, + "model": settings.llm_model, + "type": "conversation_turn", + }, + ) + except Exception as exc: + console.print(f"[yellow]Warning: failed to store memory: {exc}[/yellow]") + + # Update local conversation buffer + self.conversation.append(HumanMessage(content=message)) + self.conversation.append(response) + + return response.content + + def show_profile(self) -> None: + """Debug helper: dump the current Supermemory profile for the user.""" + result = self.memory.profile(container_tag=self.user_id, q="overview") + profile = getattr(result, "profile", None) + + console.print(Panel.fit( + f"[bold]User:[/bold] {self.user_id}\n\n" + f"[bold]Static facts:[/bold]\n{chr(10).join(profile.static or ['(none)'])}\n\n" + f"[bold]Dynamic context:[/bold]\n{chr(10).join(profile.dynamic or ['(none)'])}", + title="Supermemory Profile", + border_style="cyan", + )) + + def clear_session(self) -> None: + """Clear only the in-memory conversation (long-term memory stays in Supermemory).""" + self.conversation = [] + console.print("[yellow]Local conversation buffer cleared. Long-term memory intact.[/yellow]") diff --git a/src/thelab_langchain/cli.py b/src/thelab_langchain/cli.py new file mode 100644 index 0000000..fee64e3 --- /dev/null +++ b/src/thelab_langchain/cli.py @@ -0,0 +1,201 @@ +"""Interactive CLI for the LangChain + Supermemory demo.""" + +from __future__ import annotations + +from typing import Annotated + +import typer +from rich.console import Console +from rich.markdown import Markdown +from rich.panel import Panel +from rich.prompt import Confirm, Prompt + +from .chat import MemoryChat +from .config import settings + +# VoiceOrchestrator is imported lazily inside the `voice` command +# so that `thelab-chat chat` and `--help` do not require audio dependencies (sounddevice, etc.). + +app = typer.Typer( + name="thelab-chat", + help="LangChain + Supermemory chat demo (Grok or Claude)", + add_completion=False, +) +console = Console() + + +def _print_welcome() -> None: + base_line = "" + if settings.llm_provider == "openai_compatible" and settings.llm_base_url: + base_line = f"Base URL : [cyan]{settings.llm_base_url}[/cyan]\n" + + console.print( + Panel.fit( + "[bold green]thelab-langchain[/bold green]\n\n" + f"LLM Provider : [cyan]{settings.llm_provider}[/cyan] (model: [bold]{settings.llm_model}[/bold])\n" + f"{base_line}" + f"Memory : [magenta]Supermemory[/magenta] (container: [bold]{settings.default_user_id}[/bold])\n\n" + "Type your message and press Enter.\n" + "Special commands: [bold]/profile[/bold], [bold]/clear[/bold], [bold]/user [/bold], [bold]/quit[/bold]", + title="Memory-Aware Chat Ready", + border_style="green", + ) + ) + + +@app.command() +def chat( + user: Annotated[ + str | None, + typer.Option("--user", "-u", help="User/container ID for Supermemory isolation"), + ] = None, + model: Annotated[ + str | None, + typer.Option("--model", "-m", help="Override LLM model name"), + ] = None, +) -> None: + """Start an interactive memory-aware chat session.""" + if model: + settings.llm_model = model # type: ignore[attr-defined] + + if settings.llm_provider == "openai_compatible" and not settings.llm_base_url: + console.print( + "[yellow]Warning:[/yellow] LLM_PROVIDER=openai_compatible but no LLM_BASE_URL set.\n" + "Set LLM_BASE_URL (e.g. http://localhost:8000/v1 for local NIM on DGX Spark)." + ) + + user_id = user or settings.default_user_id + agent = MemoryChat(user_id=user_id) + + _print_welcome() + + while True: + try: + message = Prompt.ask("\n[bold blue]You[/bold blue]", console=console).strip() + except (EOFError, KeyboardInterrupt): + console.print("\n[yellow]Goodbye![/yellow]") + break + + if not message: + continue + + # Slash commands + if message.startswith("/"): + cmd = message.lower().split() + if cmd[0] in ("/quit", "/exit", "/q"): + console.print("[green]Session ended. Memories remain in Supermemory.[/green]") + break + if cmd[0] == "/profile": + agent.show_profile() + continue + if cmd[0] == "/clear": + agent.clear_session() + continue + if cmd[0] == "/user" and len(cmd) > 1: + new_user = cmd[1] + agent = MemoryChat(user_id=new_user) + console.print(f"[green]Switched to user container:[/green] [bold]{new_user}[/bold]") + continue + if cmd[0] == "/help": + console.print( + "Commands: /profile, /clear, /user , /quit\n" + "Everything else is sent to the LLM with memory context." + ) + continue + + console.print("[red]Unknown command. Try /help[/red]") + continue + + # Normal chat turn + try: + reply = agent.chat(message) + console.print("\n[bold magenta]Assistant[/bold magenta]") + console.print(Markdown(reply)) + except Exception as exc: + console.print(f"[red]Error during chat:[/red] {exc}") + if Confirm.ask("Continue session?", default=True): + continue + break + + +@app.command() +def profile(user: str = typer.Argument(None, help="User ID to inspect")) -> None: + """Print the current Supermemory profile for a user (debug).""" + user_id = user or settings.default_user_id + agent = MemoryChat(user_id=user_id) + agent.show_profile() + + +@app.command() +def env() -> None: + """Show resolved configuration (secrets redacted).""" + base_url = settings.llm_base_url or "(not set - using provider default)" + effective_key = "✓ set" if settings.effective_llm_api_key else "✗ missing" + + console.print(Panel.fit( + f"LLM Provider : {settings.llm_provider}\n" + f"LLM Model : {settings.llm_model}\n" + f"LLM Base URL : {base_url}\n" + f"Default User : {settings.default_user_id}\n" + f"XAI key : {'✓ set' if settings.xai_api_key else '✗ missing'}\n" + f"Anthropic key: {'✓ set' if settings.anthropic_api_key else '✗ missing'}\n" + f"OpenAI-comp. : {effective_key}\n" + f"Supermemory : {'✓ set' if settings.supermemory_api_key else '✗ missing'}", + title="Current Settings", + )) + + +@app.command() +def voice( + user: Annotated[str | None, typer.Option("--user", "-u", help="User/container ID for memory")] = None, + thread: Annotated[str | None, typer.Option("--thread", "-t", help="Session thread ID")] = None, + riva_uri: Annotated[str | None, typer.Option("--riva", help="Riva server address (host:port)")] = None, +) -> None: + """ + Start a voice conversation using local NVIDIA NeMo/Riva (ASR + TTS). + + This is the voice interface to the LangGraph + Supermemory brain. + Requires a running Riva server (typically on DGX Spark). + """ + user_id = user or settings.default_user_id + thread_id = thread or "voice-session" + + console.print( + Panel.fit( + f"[bold green]Voice Mode[/bold green]\n\n" + f"User : [cyan]{user_id}[/cyan]\n" + f"Thread : [cyan]{thread_id}[/cyan]\n" + f"Riva : [magenta]{riva_uri or 'localhost:50051'}[/magenta]\n\n" + "Speak naturally. Press Ctrl+C to exit.", + title="thelab-chat voice", + border_style="green", + ) + ) + + try: + # Lazy import so the rest of the CLI works without audio runtime deps + from .voice import VoiceOrchestrator + except Exception as exc: + console.print( + "[red]Voice dependencies are not available.[/red]\n" + "Install with: pip install 'thelab-langchain[voice]'\n" + "(requires libportaudio2 and a working microphone/speakers for full functionality)" + ) + raise typer.Exit(1) from exc + + try: + orchestrator = VoiceOrchestrator( + user_id=user_id, + thread_id=thread_id, + riva_uri=riva_uri, + ) + import asyncio + asyncio.run(orchestrator.start_voice_session()) + except KeyboardInterrupt: + console.print("\n[yellow]Voice session ended.[/yellow]") + except Exception as exc: + console.print(f"[red]Voice session failed:[/red] {exc}") + + +if __name__ == "__main__": + app() diff --git a/src/thelab_langchain/config.py b/src/thelab_langchain/config.py new file mode 100644 index 0000000..9a6db52 --- /dev/null +++ b/src/thelab_langchain/config.py @@ -0,0 +1,106 @@ +"""Configuration for LLM and Supermemory clients.""" + +from __future__ import annotations + +import os +from typing import Literal + +from dotenv import load_dotenv +from pydantic import Field, SecretStr +from pydantic_settings import BaseSettings, SettingsConfigDict + +load_dotenv() + + +class Settings(BaseSettings): + """Application settings loaded from environment.""" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + case_sensitive=True, + ) + + # Supermemory (required at runtime, not at import) + supermemory_api_key: SecretStr | None = Field( + default=None, alias="SUPERMEMORY_API_KEY", description="Supermemory API key" + ) + + # LLM Provider selection + llm_provider: Literal["xai", "anthropic", "openai_compatible"] = Field( + default="xai", alias="LLM_PROVIDER" + ) + + # Model names + llm_model: str = Field( + default="grok-3", + alias="LLM_MODEL", + description="Model identifier (grok-3, grok-4, claude-3-7-sonnet-20250219, ...)", + ) + + llm_temperature: float = Field(default=0.7, alias="LLM_TEMPERATURE") + llm_max_tokens: int = Field(default=2048, alias="LLM_MAX_TOKENS") + + # For local / Nemotron / any OpenAI-compatible endpoint + llm_base_url: str | None = Field( + default=None, alias="LLM_BASE_URL", + description="Base URL for OpenAI-compatible endpoints (e.g. http://localhost:8000/v1 for local NIM on DGX Spark)" + ) + + # Demo defaults + default_user_id: str = Field(default="demo-user", alias="DEFAULT_USER_ID") + + @property + def xai_api_key(self) -> SecretStr | None: + key = os.getenv("XAI_API_KEY") + return SecretStr(key) if key else None + + @property + def anthropic_api_key(self) -> SecretStr | None: + key = os.getenv("ANTHROPIC_API_KEY") + return SecretStr(key) if key else None + + @property + def openai_compatible_api_key(self) -> SecretStr | None: + """For local Nemotron / vLLM / any OpenAI-compatible server. + Many local servers accept any key or 'dummy'. + """ + key = os.getenv("OPENAI_API_KEY") or os.getenv("LLM_API_KEY") or "dummy" + return SecretStr(key) + + @property + def effective_llm_api_key(self) -> SecretStr | None: + if self.llm_provider == "xai": + return self.xai_api_key + if self.llm_provider == "anthropic": + return self.anthropic_api_key + if self.llm_provider == "openai_compatible": + return self.openai_compatible_api_key + return None + + def validate_keys(self) -> None: + """Ensure the required API key for the chosen provider is present.""" + if self.llm_provider == "xai" and not self.xai_api_key: + raise ValueError( + "XAI_API_KEY is required when LLM_PROVIDER=xai. " + "Get one at https://console.x.ai/" + ) + if self.llm_provider == "anthropic" and not self.anthropic_api_key: + raise ValueError( + "ANTHROPIC_API_KEY is required when LLM_PROVIDER=anthropic. " + "Get one at https://console.anthropic.com/" + ) + if self.llm_provider == "openai_compatible" and not self.llm_base_url: + raise ValueError( + "LLM_BASE_URL is required when LLM_PROVIDER=openai_compatible " + "(e.g. http://nemotron:8000/v1 for local Nemotron NIM)." + ) + if not self.supermemory_api_key: + raise ValueError( + "SUPERMEMORY_API_KEY is required. " + "Get one at https://console.supermemory.ai/" + ) + + +settings = Settings() diff --git a/src/thelab_langchain/llm.py b/src/thelab_langchain/llm.py new file mode 100644 index 0000000..e949a5c --- /dev/null +++ b/src/thelab_langchain/llm.py @@ -0,0 +1,62 @@ +""" +LLM factory for the TheLab agent. + +Supports: +- xai (Grok via langchain-xai) +- anthropic (Claude via langchain-anthropic) +- openai_compatible (Nemotron NIM, vLLM, Ollama, etc. via langchain-openai) + +This is the single place that knows how to construct the right chat model +based on environment configuration. +""" + +from __future__ import annotations + +from langchain_core.language_models import BaseChatModel + +from .config import settings + + +def get_chat_model() -> BaseChatModel: + """ + Returns the configured chat model based on LLM_PROVIDER. + """ + if settings.llm_provider == "xai": + from langchain_xai import ChatXAI + + return ChatXAI( + model=settings.llm_model, + temperature=settings.llm_temperature, + max_tokens=settings.llm_max_tokens, + api_key=settings.xai_api_key.get_secret_value() if settings.xai_api_key else None, + ) + + if settings.llm_provider == "anthropic": + from langchain_anthropic import ChatAnthropic + + return ChatAnthropic( + model=settings.llm_model, + temperature=settings.llm_temperature, + max_tokens=settings.llm_max_tokens, + api_key=settings.anthropic_api_key.get_secret_value() if settings.anthropic_api_key else None, + ) + + if settings.llm_provider == "openai_compatible": + from langchain_openai import ChatOpenAI + + # For local Nemotron / vLLM / NIMs, the key is often "dummy" + api_key = ( + settings.openai_compatible_api_key.get_secret_value() + if settings.openai_compatible_api_key + else "dummy" + ) + + return ChatOpenAI( + model=settings.llm_model, + temperature=settings.llm_temperature, + max_tokens=settings.llm_max_tokens, + base_url=settings.llm_base_url, + api_key=api_key, + ) + + raise ValueError(f"Unsupported llm_provider: {settings.llm_provider}") diff --git a/src/thelab_langchain/voice/__init__.py b/src/thelab_langchain/voice/__init__.py new file mode 100644 index 0000000..b241981 --- /dev/null +++ b/src/thelab_langchain/voice/__init__.py @@ -0,0 +1,24 @@ +""" +Voice layer for the TheLab agent on DGX Spark. + +This package handles: +- Local NVIDIA NeMo / Riva ASR (STT) and TTS +- Audio I/O (microphone / speakers) +- Turn management and barge-in +- Bridging between audio and the LangGraph agent brain +""" + +from .audio import AudioConfig, play_audio, record_until_silence +from .orchestrator import VoiceOrchestrator +from .riva import RivaASR, RivaConfig, RivaTTS, get_riva_clients + +__all__ = [ + "AudioConfig", + "play_audio", + "record_until_silence", + "VoiceOrchestrator", + "RivaASR", + "RivaConfig", + "RivaTTS", + "get_riva_clients", +] diff --git a/src/thelab_langchain/voice/audio.py b/src/thelab_langchain/voice/audio.py new file mode 100644 index 0000000..4b30344 --- /dev/null +++ b/src/thelab_langchain/voice/audio.py @@ -0,0 +1,114 @@ +""" +Basic audio I/O helpers using sounddevice. + +Provides: +- Record audio from microphone until silence (simple energy VAD) +- Play audio to speakers +- Utility functions for normalization + +This is intentionally simple for the Phase 1 MVP. +We can upgrade to webrtcvad or Silero VAD + proper streaming later. +""" + +from __future__ import annotations + +import numpy as np +import sounddevice as sd + + +class AudioConfig: + """Audio settings for the voice pipeline.""" + + sample_rate: int = 16000 # Good for both ASR and many TTS voices + channels: int = 1 + dtype: str = "float32" # We work in float32 [-1, 1] + chunk_duration: float = 0.1 # How often we check for silence (seconds) + silence_threshold: float = 0.01 # RMS energy below this = silence + silence_duration: float = 0.8 # How long silence before we stop recording + max_record_seconds: float = 30.0 # Safety cap + + +def record_until_silence( + config: AudioConfig | None = None, +) -> np.ndarray: + """ + Record from the default microphone until the user stops speaking. + + Uses a very simple energy-based VAD (RMS). + + Returns: + 1D float32 numpy array of the recorded audio (normalized [-1, 1]). + """ + cfg = config or AudioConfig() + + print("[Audio] Listening... (speak now)") + + frames: list[np.ndarray] = [] + silence_counter = 0 + max_chunks = int(cfg.max_record_seconds / cfg.chunk_duration) + + stream = sd.InputStream( + samplerate=cfg.sample_rate, + channels=cfg.channels, + dtype=cfg.dtype, + ) + stream.start() + + try: + for _ in range(max_chunks): + audio_chunk, _ = stream.read(int(cfg.sample_rate * cfg.chunk_duration)) + audio_chunk = audio_chunk.flatten() + + # Simple RMS energy + rms = np.sqrt(np.mean(audio_chunk**2)) + + frames.append(audio_chunk) + + if rms < cfg.silence_threshold: + silence_counter += 1 + else: + silence_counter = 0 + + if silence_counter * cfg.chunk_duration >= cfg.silence_duration: + break + finally: + stream.stop() + stream.close() + + if not frames: + return np.array([], dtype=np.float32) + + full_audio = np.concatenate(frames) + print(f"[Audio] Captured {len(full_audio) / cfg.sample_rate:.1f}s of audio") + return full_audio + + +def play_audio( + audio: np.ndarray, + sample_rate: int = 16000, + blocking: bool = True, +) -> None: + """ + Play audio through the default output device. + + Args: + audio: 1D float32 array in [-1.0, 1.0] + sample_rate: Sample rate of the audio + blocking: If True, wait until playback finishes + """ + if len(audio) == 0: + return + + print("[Audio] Playing response...") + sd.play(audio, samplerate=sample_rate) + if blocking: + sd.wait() + print("[Audio] Playback finished") + + +def get_default_devices() -> dict: + """Return info about the current default input/output devices (useful for debugging).""" + return { + "input": sd.query_devices(kind="input"), + "output": sd.query_devices(kind="output"), + } diff --git a/src/thelab_langchain/voice/orchestrator.py b/src/thelab_langchain/voice/orchestrator.py new file mode 100644 index 0000000..ba2bf6d --- /dev/null +++ b/src/thelab_langchain/voice/orchestrator.py @@ -0,0 +1,208 @@ +""" +VoiceOrchestrator - Bridges audio (via NVIDIA Riva) with the LangGraph agent brain. + +This is the core of the voice experience. It manages: +- Listening for user speech (ASR) +- Sending transcribed text to the agent +- Receiving responses and synthesizing speech (TTS) +- Handling interruptions (barge-in) +""" + +from __future__ import annotations + +import json +import os +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from langchain_core.messages import HumanMessage + +from thelab_langchain.agent.graph import get_agent # type: ignore + +from .audio import AudioConfig, play_audio, record_until_silence +from .riva import get_riva_clients + + +class VoiceOrchestrator: + """ + High-level coordinator for voice conversations. + + Phase 1 (MVP) implementation: + - Record until silence (energy VAD) + - Transcribe with Riva ASR + - Call the agent brain + - Synthesize response with Riva TTS + - Play audio + + Phase 2 will upgrade this to true streaming + barge-in. + """ + + def __init__( + self, + user_id: str = "default", + thread_id: str = "default", + agent: Callable[[str], str] | None = None, + riva_uri: str | None = None, + ): + self.user_id = user_id + self.thread_id = thread_id + + # Prefer the real LangGraph agent if none is passed in + if agent is None: + try: + compiled_graph = get_agent(user_id=self.user_id) + self.agent = lambda text: self._call_langgraph_agent(text, compiled_graph) + except Exception: + self.agent = self._default_agent + else: + self.agent = agent + + # Riva clients (can point to local Riva server on DGX Spark) + self.asr, self.tts = get_riva_clients(uri=riva_uri) + + self.audio_config = AudioConfig() + self._running = False + + # Benchmark instrumentation (enabled when BENCHMARK_MODE=1 or BENCHMARK_REPORT_DIR is set) + self._benchmark_mode = bool(os.getenv("BENCHMARK_MODE") or os.getenv("BENCHMARK_REPORT_DIR")) + self._benchmark_report_dir: Path | None = None + if self._benchmark_mode: + report_dir = os.getenv("BENCHMARK_REPORT_DIR") + if report_dir: + self._benchmark_report_dir = Path(report_dir) + self._benchmark_report_dir.mkdir(parents=True, exist_ok=True) + + def _emit_benchmark_event(self, event: dict[str, Any]) -> None: + """Emit a structured timing / measurement event (no-op unless benchmark mode).""" + if not self._benchmark_mode: + return + + event = {"ts": time.time(), "type": event.get("type", "event"), **event} + + # Always print a machine-readable line (easy to capture in baseline runs) + print(f"[BENCHMARK] {json.dumps(event, default=str)}") + + # If a report dir was provided, append to events.jsonl there as well + if self._benchmark_report_dir: + events_file = self._benchmark_report_dir / "events.jsonl" + with events_file.open("a") as f: + f.write(json.dumps(event, default=str) + "\n") + + def _default_agent(self, text: str) -> str: + """Very basic fallback (used only if the real graph fails to load).""" + return f"You said: {text}. (Using fallback agent — real LangGraph brain not wired yet.)" + + def _call_langgraph_agent(self, text: str, graph) -> str: + """Call the compiled LangGraph and extract the final response.""" + try: + # Pass a proper HumanMessage so the graph state stays clean + result = graph.invoke({ + "messages": [HumanMessage(content=text)], + "user_id": self.user_id, + "thread_id": self.thread_id, + }) + + if isinstance(result, dict) and "messages" in result: + last_msg = result["messages"][-1] + if hasattr(last_msg, "content"): + return last_msg.content + return str(last_msg) + return str(result) + except Exception as e: + return f"[Agent error] {e}" + + async def start_voice_session(self) -> None: + """Run the main voice conversation loop until stopped.""" + print("[Voice] Starting voice session (MVP non-streaming mode)") + print(f"[Voice] User: {self.user_id} | Thread: {self.thread_id}") + print("[Voice] Say something... (Ctrl+C to stop)") + + self._running = True + + try: + while self._running: + # 1. Listen (end of speech = VAD trigger) + eos_start = time.time() + audio = record_until_silence(self.audio_config) + eos_end = time.time() + + if len(audio) < self.audio_config.sample_rate * 0.3: + # Too short, probably noise — ignore + continue + + self._emit_benchmark_event({ + "type": "end_of_speech", + "duration_s": round(eos_end - eos_start, 3), + "audio_length_s": round(len(audio) / self.audio_config.sample_rate, 2), + }) + + # 2. Transcribe + t0 = time.time() + transcript = self.asr.recognize(audio, self.audio_config.sample_rate) + t1 = time.time() + + if not transcript: + print("[Voice] (no speech detected)") + continue + + print(f"\n[You] {transcript}") + + self._emit_benchmark_event({ + "type": "transcript_ready", + "asr_latency_s": round(t1 - t0, 3), + "transcript_len": len(transcript), + }) + + # 3. Call the agent brain + t0 = time.time() + response_text = self.agent(transcript) + t1 = time.time() + + print(f"[Agent] {response_text}") + + self._emit_benchmark_event({ + "type": "response_ready", + "agent_latency_s": round(t1 - t0, 3), + "response_len": len(response_text or ""), + }) + + # 4. Synthesize + play (first audio out approximation) + t0 = time.time() + tts_audio = self.tts.synthesize( + response_text, + sample_rate=22050, # Common for many NeMo TTS voices + ) + t1 = time.time() + + play_start = time.time() + play_audio(tts_audio, sample_rate=22050) + play_end = time.time() + + self._emit_benchmark_event({ + "type": "first_audio_out", + "tts_latency_s": round(t1 - t0, 3), + "playback_duration_s": round(play_end - play_start, 3), + "total_turn_latency_s": round(play_start - eos_end, 3), # rough end-to-end voice turn + }) + + except KeyboardInterrupt: + print("\n[Voice] Session interrupted by user") + finally: + await self.stop() + + async def stop(self) -> None: + """Stop the voice session.""" + self._running = False + print("[Voice] Session ended") + + # --- Future hooks for streaming / barge-in (Phase 2) --- + + async def _listen_streaming(self): + """Placeholder for true streaming ASR.""" + raise NotImplementedError("Coming in Phase 2") + + async def _speak_streaming(self, text: str): + """Placeholder for streaming TTS (start speaking before full response).""" + raise NotImplementedError("Coming in Phase 2") diff --git a/src/thelab_langchain/voice/riva.py b/src/thelab_langchain/voice/riva.py new file mode 100644 index 0000000..0f1476e --- /dev/null +++ b/src/thelab_langchain/voice/riva.py @@ -0,0 +1,125 @@ +""" +NVIDIA Riva client wrappers for ASR (STT) and TTS. + +This module provides a clean interface over the official `nvidia-riva-client` +so the rest of the voice layer doesn't have to know gRPC details. + +Designed for local Riva / NeMo deployment on DGX Spark. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +import numpy as np +import riva.client +from riva.client import ( + ASRService, + Auth, + RecognitionConfig, + SpeechSynthesisService, +) + + +@dataclass +class RivaConfig: + """Configuration for connecting to a Riva server.""" + + uri: str = "localhost:50051" + ssl: bool = False + language_code: str = "en-US" + # ASR + asr_model: str = "" # leave empty for server default + # TTS + tts_voice: str = "" # e.g. "English-US.Female-1" or leave empty for default + + +def _make_auth(cfg: RivaConfig) -> Auth: + return Auth(uri=cfg.uri, use_ssl=cfg.ssl) + + +class RivaASR: + """Wrapper around Riva ASR (speech-to-text).""" + + def __init__(self, config: RivaConfig | None = None): + self.config = config or RivaConfig() + auth = _make_auth(self.config) + self._client = ASRService(auth) + + def recognize( + self, + audio: np.ndarray, + sample_rate: int = 16000, + ) -> str: + """ + Transcribe a full audio buffer (non-streaming). + + Args: + audio: 1D float32 numpy array in [-1.0, 1.0] or int16. + sample_rate: Sample rate of the audio (Riva typically expects 16000). + + Returns: + The transcribed text. + """ + if audio.dtype != np.int16: + # Convert float32 [-1,1] → int16 + audio = (audio * 32767).astype(np.int16) + + config = RecognitionConfig( + encoding=riva.client.AudioEncoding.LINEAR_PCM, + sample_rate_hertz=sample_rate, + language_code=self.config.language_code, + model=self.config.asr_model or None, + max_alternatives=1, + enable_automatic_punctuation=True, + ) + + response = self._client.offline_recognize(audio, config) + if response.results: + return response.results[0].alternatives[0].transcript.strip() + return "" + + # TODO (Phase 2): Add streaming_recognize() for real-time partial results + + +class RivaTTS: + """Wrapper around Riva TTS (text-to-speech).""" + + def __init__(self, config: RivaConfig | None = None): + self.config = config or RivaConfig() + auth = _make_auth(self.config) + self._client = SpeechSynthesisService(auth) + + def synthesize( + self, + text: str, + sample_rate: int = 22050, + ) -> np.ndarray: + """ + Convert text to audio waveform. + + Returns: + 1D float32 numpy array normalized to [-1.0, 1.0] + """ + response = self._client.synthesize( + text=text, + voice_name=self.config.tts_voice or None, + language_code=self.config.language_code, + sample_rate_hertz=sample_rate, + encoding=riva.client.AudioEncoding.LINEAR_PCM, + ) + + audio = np.frombuffer(response.audio, dtype=np.int16) + # Normalize to float32 [-1.0, 1.0] + return audio.astype(np.float32) / 32768.0 + + +# Convenience factory +def get_riva_clients( + uri: str | None = None, +) -> tuple[RivaASR, RivaTTS]: + """Create both ASR and TTS clients from environment or defaults.""" + uri = uri or os.getenv("RIVA_URI", "localhost:50051") + cfg = RivaConfig(uri=uri) + return RivaASR(cfg), RivaTTS(cfg) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_agent_graph.py b/tests/test_agent_graph.py new file mode 100644 index 0000000..08078de --- /dev/null +++ b/tests/test_agent_graph.py @@ -0,0 +1,107 @@ +"""Tests for the LangGraph agent brain: routing and memory injection. + +These are pure-logic tests. All external services (Supermemory and the LLM) are +mocked, so the suite needs no network access, API keys, GPU, or audio libraries. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage +from langgraph.graph import END + +from thelab_langchain.agent import graph +from thelab_langchain.agent.graph import _should_continue +from thelab_langchain.agent.state import AgentState + + +def _state(*messages) -> AgentState: + return AgentState(user_id="test-user", messages=list(messages)) + + +# --- _should_continue ------------------------------------------------------- + + +def test_should_continue_routes_to_tools_when_tool_calls_present(): + ai = AIMessage( + content="", + tool_calls=[ + {"name": "recall_memories", "args": {"query": "hi"}, "id": "call-1", "type": "tool_call"} + ], + ) + assert _should_continue(_state(HumanMessage(content="hi"), ai)) == "execute_tools" + + +def test_should_continue_ends_when_no_tool_calls(): + ai = AIMessage(content="all done, nothing to look up") + assert _should_continue(_state(HumanMessage(content="hi"), ai)) == END + + +def test_should_continue_ends_on_non_ai_message(): + # A HumanMessage as the last message must not be routed to tools. + assert _should_continue(_state(HumanMessage(content="still my turn"))) == END + + +# --- _memory_injection ------------------------------------------------------ + + +class _FakeTool: + """Stand-in for a bound Supermemory LangChain tool.""" + + def __init__(self, name: str, result: str) -> None: + self.name = name + self._result = result + self.calls: list = [] + + def invoke(self, arg): + self.calls.append(arg) + return self._result + + +def test_memory_injection_injects_system_message_without_summarization_llm(monkeypatch): + profile_tool = _FakeTool("get_user_profile", "## User Profile\n- Likes strong coffee") + recall_tool = _FakeTool("recall_memories", "- Talked about the DGX Spark voice agent") + store_tool = _FakeTool("store_memory", "") + + monkeypatch.setattr( + graph, "create_memory_tools", lambda user_id: [recall_tool, store_tool, profile_tool] + ) + + # If the node ever tried to summarize memory via an LLM round-trip, this fires. + llm_factory = MagicMock() + monkeypatch.setattr(graph, "get_chat_model", llm_factory) + + state = _state(HumanMessage(content="What do you know about my project?")) + result = graph._memory_injection(state) + + messages = result["messages"] + injected = messages[0] + assert isinstance(injected, SystemMessage) + assert "User Context (from long-term memory)" in injected.content + assert "Likes strong coffee" in injected.content + assert "DGX Spark voice agent" in injected.content + + # The original conversation is preserved after the injected context. + assert isinstance(messages[-1], HumanMessage) + + # The node fetched from Supermemory (profile + recall). + assert profile_tool.calls + assert recall_tool.calls + + # Latency design point: NO extra summarization LLM round-trip per voice turn. + llm_factory.assert_not_called() + + +def test_memory_injection_returns_empty_when_no_context(monkeypatch): + empty_profile = _FakeTool("get_user_profile", "") + empty_recall = _FakeTool("recall_memories", "") + monkeypatch.setattr(graph, "create_memory_tools", lambda user_id: [empty_recall, empty_profile]) + + llm_factory = MagicMock() + monkeypatch.setattr(graph, "get_chat_model", llm_factory) + + result = graph._memory_injection(_state(HumanMessage(content="hello"))) + + assert result == {} + llm_factory.assert_not_called() diff --git a/tests/test_chat.py b/tests/test_chat.py new file mode 100644 index 0000000..1f67fc3 --- /dev/null +++ b/tests/test_chat.py @@ -0,0 +1,29 @@ +"""Tests for MemoryContext.to_prompt_block formatting (pure, no services).""" + +from __future__ import annotations + +from thelab_langchain.chat import MemoryContext + + +def test_to_prompt_block_empty_context(): + block = MemoryContext().to_prompt_block() + + assert block.startswith("## User Memory Context") + assert "No prior memories or profile for this user yet." in block + + +def test_to_prompt_block_with_all_fields(): + ctx = MemoryContext( + static_facts=["Prefers concise answers"], + dynamic_context=["Building a voice agent on DGX Spark"], + relevant_memories=["Discussed Supermemory integration last week"], + ) + block = ctx.to_prompt_block() + + assert "### Long-term Profile" in block + assert "Prefers concise answers" in block + assert "### Recent Activity" in block + assert "Building a voice agent on DGX Spark" in block + assert "### Relevant Past Memories" in block + assert "Discussed Supermemory integration last week" in block + assert "No prior memories" not in block diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..68073bd --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,60 @@ +"""Tests for Settings.validate_keys. + +Pure configuration logic. `_env_file=None` keeps each test hermetic (the working +tree's real .env is ignored); provider selection and keys are driven entirely by +monkeypatched environment variables. +""" + +from __future__ import annotations + +import pytest + +from thelab_langchain.config import Settings + + +def _settings() -> Settings: + return Settings(_env_file=None) + + +def test_validate_keys_ok_for_xai(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "xai") + monkeypatch.setenv("XAI_API_KEY", "xai-test-key") + monkeypatch.setenv("SUPERMEMORY_API_KEY", "sm-test-key") + + _settings().validate_keys() # should not raise + + +def test_validate_keys_missing_xai_key(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "xai") + monkeypatch.setenv("SUPERMEMORY_API_KEY", "sm-test-key") + monkeypatch.delenv("XAI_API_KEY", raising=False) + + with pytest.raises(ValueError, match="XAI_API_KEY"): + _settings().validate_keys() + + +def test_validate_keys_missing_anthropic_key(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("SUPERMEMORY_API_KEY", "sm-test-key") + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + + with pytest.raises(ValueError, match="ANTHROPIC_API_KEY"): + _settings().validate_keys() + + +def test_validate_keys_openai_compatible_requires_base_url(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "openai_compatible") + monkeypatch.setenv("SUPERMEMORY_API_KEY", "sm-test-key") + monkeypatch.delenv("LLM_BASE_URL", raising=False) + + with pytest.raises(ValueError, match="LLM_BASE_URL"): + _settings().validate_keys() + + +def test_validate_keys_missing_supermemory_key(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "xai") + monkeypatch.setenv("XAI_API_KEY", "xai-test-key") + monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False) + + with pytest.raises(ValueError, match="SUPERMEMORY_API_KEY"): + _settings().validate_keys() diff --git a/tests/test_llm.py b/tests/test_llm.py new file mode 100644 index 0000000..b96c4a5 --- /dev/null +++ b/tests/test_llm.py @@ -0,0 +1,68 @@ +"""Tests for get_chat_model provider routing. + +Fake provider modules are injected into sys.modules so that no real LangChain +provider package (langchain-xai / langchain-anthropic / langchain-openai) needs +to be installed and no client ever makes a network call. +""" + +from __future__ import annotations + +import sys +import types +from unittest.mock import MagicMock + +import pytest + +from thelab_langchain import config, llm + + +def _fake_provider(module_name: str, class_name: str): + """Register a fake provider module and return (module, fake_class).""" + module = types.ModuleType(module_name) + fake_class = MagicMock(name=class_name) + setattr(module, class_name, fake_class) + return module, fake_class + + +def test_get_chat_model_routes_to_xai(monkeypatch): + module, chat_xai = _fake_provider("langchain_xai", "ChatXAI") + monkeypatch.setitem(sys.modules, "langchain_xai", module) + monkeypatch.setattr(config.settings, "llm_provider", "xai") + + result = llm.get_chat_model() + + assert result is chat_xai.return_value + chat_xai.assert_called_once() + assert chat_xai.call_args.kwargs["model"] == config.settings.llm_model + + +def test_get_chat_model_routes_to_anthropic(monkeypatch): + module, chat_anthropic = _fake_provider("langchain_anthropic", "ChatAnthropic") + monkeypatch.setitem(sys.modules, "langchain_anthropic", module) + monkeypatch.setattr(config.settings, "llm_provider", "anthropic") + + result = llm.get_chat_model() + + assert result is chat_anthropic.return_value + chat_anthropic.assert_called_once() + + +def test_get_chat_model_routes_to_openai_compatible(monkeypatch): + module, chat_openai = _fake_provider("langchain_openai", "ChatOpenAI") + monkeypatch.setitem(sys.modules, "langchain_openai", module) + monkeypatch.setattr(config.settings, "llm_provider", "openai_compatible") + monkeypatch.setattr(config.settings, "llm_base_url", "http://localhost:8000/v1") + + result = llm.get_chat_model() + + assert result is chat_openai.return_value + kwargs = chat_openai.call_args.kwargs + assert kwargs["base_url"] == "http://localhost:8000/v1" + # Local OpenAI-compatible servers accept any key; a non-empty fallback is used. + assert kwargs["api_key"] + + +def test_get_chat_model_unknown_provider_raises(monkeypatch): + monkeypatch.setattr(config.settings, "llm_provider", "does-not-exist") + with pytest.raises(ValueError, match="Unsupported llm_provider"): + llm.get_chat_model()