diff --git a/.devcontainer/building-comfort/devcontainer-lock.json b/.devcontainer/building-comfort/devcontainer-lock.json new file mode 100644 index 0000000..895e5a2 --- /dev/null +++ b/.devcontainer/building-comfort/devcontainer-lock.json @@ -0,0 +1,24 @@ +{ + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "version": "2.5.9", + "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a", + "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "2.17.0", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c", + "integrity": "sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c" + }, + "ghcr.io/devcontainers/features/git:1": { + "version": "1.3.8", + "resolved": "ghcr.io/devcontainers/features/git@sha256:fd75977de13a9979000e0e78baf949adb0ca71d2398995fa22e0a36d7e7e7fe2", + "integrity": "sha256:fd75977de13a9979000e0e78baf949adb0ca71d2398995fa22e0a36d7e7e7fe2" + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "1.8.0", + "resolved": "ghcr.io/devcontainers/features/python@sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511", + "integrity": "sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511" + } + } +} diff --git a/.devcontainer/building-comfort/devcontainer.json b/.devcontainer/building-comfort/devcontainer.json new file mode 100644 index 0000000..349dab7 --- /dev/null +++ b/.devcontainer/building-comfort/devcontainer.json @@ -0,0 +1,60 @@ +{ + "name": "Building Comfort Tutorial (Python)", + "image": "mcr.microsoft.com/devcontainers/base:bookworm@sha256:9e35ebf156350ba25b117da6869907034b6e2437b1891e3a43395af0b7d3eda4", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "29", + "moby": false + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12", + "installTools": true + }, + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "configureZshAsDefaultShell": true + } + }, + "postCreateCommand": "bash ../../.devcontainer/building-comfort/post-create.sh", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}/tutorials/building-comfort", + "runArgs": [ + "--privileged", + "--init", + "--ipc=host" + ], + "forwardPorts": [ + 8501, + 5732 + ], + "portsAttributes": { + "8501": { + "label": "Streamlit UI", + "onAutoForward": "notify" + }, + "5732": { + "label": "PostgreSQL", + "onAutoForward": "silent" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python" + ], + "settings": { + "editor.formatOnSave": true + } + } + }, + "remoteUser": "vscode", + "remoteEnv": { + "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" + }, + "containerEnv": { + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5732", + "POSTGRES_HOST_PORT": "5732", + "STREAMLIT_PORT": "8501" + } +} diff --git a/.devcontainer/building-comfort/post-create.sh b/.devcontainer/building-comfort/post-create.sh new file mode 100755 index 0000000..fd64fb1 --- /dev/null +++ b/.devcontainer/building-comfort/post-create.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Post-create script for the Building Comfort tutorial (Python). + +set -e + +echo "🔧 Initializing the Building Comfort (Python) tutorial environment..." + +# Resolve the tutorial directory from this script's location so the script works +# regardless of the current working directory. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +TUTORIAL_DIR="$REPO_ROOT/tutorials/building-comfort" + +# Install the tutorial's Python dependencies (drasi-lib, streamlit, psycopg). +# The devcontainer's Python feature provides a pip that is not externally +# managed, so a plain install works and puts `streamlit` on PATH. +echo "🐍 Installing Python dependencies..." +cd "$TUTORIAL_DIR" +python -m pip install --upgrade pip +python -m pip install -r requirements.txt + +echo "" +echo "✅ Building Comfort (Python) tutorial environment is ready!" +echo " Next: run 'bash scripts/start-demo.sh' (you are already in tutorials/building-comfort)" diff --git a/.devcontainer/curbside-pickup/devcontainer-lock.json b/.devcontainer/curbside-pickup/devcontainer-lock.json new file mode 100644 index 0000000..895e5a2 --- /dev/null +++ b/.devcontainer/curbside-pickup/devcontainer-lock.json @@ -0,0 +1,24 @@ +{ + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "version": "2.5.9", + "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a", + "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "2.17.0", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c", + "integrity": "sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c" + }, + "ghcr.io/devcontainers/features/git:1": { + "version": "1.3.8", + "resolved": "ghcr.io/devcontainers/features/git@sha256:fd75977de13a9979000e0e78baf949adb0ca71d2398995fa22e0a36d7e7e7fe2", + "integrity": "sha256:fd75977de13a9979000e0e78baf949adb0ca71d2398995fa22e0a36d7e7e7fe2" + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "1.8.0", + "resolved": "ghcr.io/devcontainers/features/python@sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511", + "integrity": "sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511" + } + } +} diff --git a/.devcontainer/curbside-pickup/devcontainer.json b/.devcontainer/curbside-pickup/devcontainer.json new file mode 100644 index 0000000..cd95247 --- /dev/null +++ b/.devcontainer/curbside-pickup/devcontainer.json @@ -0,0 +1,68 @@ +{ + "name": "Curbside Pickup Tutorial (Python)", + "image": "mcr.microsoft.com/devcontainers/base:bookworm@sha256:9e35ebf156350ba25b117da6869907034b6e2437b1891e3a43395af0b7d3eda4", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "29", + "moby": false + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12", + "installTools": true + }, + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "configureZshAsDefaultShell": true + } + }, + "postCreateCommand": "bash ../../.devcontainer/curbside-pickup/post-create.sh", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}/tutorials/curbside-pickup", + "runArgs": [ + "--privileged", + "--init", + "--ipc=host" + ], + "forwardPorts": [ + 8501, + 5742, + 3309 + ], + "portsAttributes": { + "8501": { + "label": "Streamlit UI", + "onAutoForward": "notify" + }, + "5742": { + "label": "PostgreSQL (orders)", + "onAutoForward": "silent" + }, + "3309": { + "label": "MySQL (vehicles)", + "onAutoForward": "silent" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python" + ], + "settings": { + "editor.formatOnSave": true + } + } + }, + "remoteUser": "vscode", + "remoteEnv": { + "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" + }, + "containerEnv": { + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5742", + "POSTGRES_HOST_PORT": "5742", + "MYSQL_HOST": "localhost", + "MYSQL_PORT": "3309", + "MYSQL_HOST_PORT": "3309", + "STREAMLIT_PORT": "8501" + } +} diff --git a/.devcontainer/curbside-pickup/post-create.sh b/.devcontainer/curbside-pickup/post-create.sh new file mode 100755 index 0000000..3dac6b6 --- /dev/null +++ b/.devcontainer/curbside-pickup/post-create.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Post-create script for the Curbside Pickup tutorial (Python). + +set -e + +echo "🔧 Initializing the Curbside Pickup (Python) tutorial environment..." + +# Resolve the tutorial directory from this script's location so the script works +# regardless of the current working directory. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +TUTORIAL_DIR="$REPO_ROOT/tutorials/curbside-pickup" + +# Install the tutorial's Python dependencies (drasi-lib, streamlit, psycopg, +# PyMySQL). The devcontainer's Python feature provides a pip that is not +# externally managed, so a plain install works and puts `streamlit` on PATH. +echo "🐍 Installing Python dependencies..." +cd "$TUTORIAL_DIR" +python -m pip install --upgrade pip +python -m pip install -r requirements.txt + +echo "" +echo "✅ Curbside Pickup (Python) tutorial environment is ready!" +echo " Next: run 'bash scripts/start-demo.sh' (you are already in tutorials/curbside-pickup)" diff --git a/.devcontainer/getting-started/devcontainer-lock.json b/.devcontainer/getting-started/devcontainer-lock.json new file mode 100644 index 0000000..895e5a2 --- /dev/null +++ b/.devcontainer/getting-started/devcontainer-lock.json @@ -0,0 +1,24 @@ +{ + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "version": "2.5.9", + "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a", + "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "2.17.0", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c", + "integrity": "sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c" + }, + "ghcr.io/devcontainers/features/git:1": { + "version": "1.3.8", + "resolved": "ghcr.io/devcontainers/features/git@sha256:fd75977de13a9979000e0e78baf949adb0ca71d2398995fa22e0a36d7e7e7fe2", + "integrity": "sha256:fd75977de13a9979000e0e78baf949adb0ca71d2398995fa22e0a36d7e7e7fe2" + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "1.8.0", + "resolved": "ghcr.io/devcontainers/features/python@sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511", + "integrity": "sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511" + } + } +} diff --git a/.devcontainer/getting-started/devcontainer.json b/.devcontainer/getting-started/devcontainer.json new file mode 100644 index 0000000..801b278 --- /dev/null +++ b/.devcontainer/getting-started/devcontainer.json @@ -0,0 +1,54 @@ +{ + "name": "Getting Started Tutorial (Python)", + "image": "mcr.microsoft.com/devcontainers/base:bookworm@sha256:9e35ebf156350ba25b117da6869907034b6e2437b1891e3a43395af0b7d3eda4", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "29", + "moby": false + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12", + "installTools": true + }, + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "configureZshAsDefaultShell": true + } + }, + "postCreateCommand": "bash ../../.devcontainer/getting-started/post-create.sh", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}/tutorials/getting-started", + "runArgs": [ + "--privileged", + "--init", + "--ipc=host" + ], + "forwardPorts": [ + 5752 + ], + "portsAttributes": { + "5752": { + "label": "PostgreSQL", + "onAutoForward": "silent" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python" + ], + "settings": { + "editor.formatOnSave": true + } + } + }, + "remoteUser": "vscode", + "remoteEnv": { + "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" + }, + "containerEnv": { + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5752", + "POSTGRES_HOST_PORT": "5752" + } +} diff --git a/.devcontainer/getting-started/post-create.sh b/.devcontainer/getting-started/post-create.sh new file mode 100755 index 0000000..701dc00 --- /dev/null +++ b/.devcontainer/getting-started/post-create.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Post-create script for the Getting Started tutorial (Python). + +set -e + +echo "🔧 Initializing the Getting Started (Python) tutorial environment..." + +# Resolve the tutorial directory from this script's location so the script works +# regardless of the current working directory. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +TUTORIAL_DIR="$REPO_ROOT/tutorials/getting-started" + +# Install the tutorial's Python dependency (drasi-lib). The devcontainer's Python +# feature provides a pip that is not externally managed, so a plain install works. +echo "🐍 Installing Python dependencies..." +cd "$TUTORIAL_DIR" +python -m pip install --upgrade pip +python -m pip install -r requirements.txt + +echo "" +echo "✅ Getting Started (Python) tutorial environment is ready!" +echo " Next: run 'bash scripts/start-demo.sh' (you are already in tutorials/getting-started)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11f753f..537437e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,9 @@ jobs: - name: Python formatting run: .venv/bin/ruff format --check . + - name: Tutorial READMEs are current + run: python3 scripts/render-tutorials.py --check + # This went unrun for long enough to accumulate 1653 errors from a single # misconfiguration, which is exactly the kind of rot a gate is meant to # prevent. pyright resolves `drasi` from the source tree, so the diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 9774484..cb4bd6c 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -12,11 +12,13 @@ on: branches: [main] paths: - "website/**" + - "tutorials/**" - ".github/workflows/website.yml" pull_request: branches: [main] paths: - "website/**" + - "tutorials/**" - ".github/workflows/website.yml" # Allow the deploy job to publish to GitHub Pages via OIDC. diff --git a/README.md b/README.md index d6d5422..34f5728 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,14 @@ make venv && make develop .venv/bin/python examples/python_source.py ``` +## Tutorials + +Longer, hands-on walkthroughs live in [`tutorials/`](https://github.com/drasi-project/drasi-python/tree/main/tutorials). +The [Building Comfort](https://github.com/drasi-project/drasi-python/tree/main/tutorials/building-comfort) +tutorial builds a smart-building monitor from PostgreSQL CDC, six continuous +queries with synthetic joins, and a Python reaction that drives a live Streamlit +UI. The tutorials also render on the [documentation site](https://drasi-project.github.io/drasi-python/docs/tutorials/). + ## Development ```bash diff --git a/scripts/render-tutorials.py b/scripts/render-tutorials.py new file mode 100644 index 0000000..838ce77 --- /dev/null +++ b/scripts/render-tutorials.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render Docsy/Hugo tutorial sources into plain Markdown READMEs. + +Each tutorial is authored once in an ``_index.md`` that may use Docsy/Hugo +shortcodes (``{{< tabpane >}}``, ``{{% alert %}}``, ...). The doc site consumes +``_index.md`` directly (mounted into the Hugo content tree) so the tab widgets +and styled alerts render. GitHub and plain Markdown viewers cannot process +shortcodes, so this script generates a sibling ``README.md`` with the shortcodes +converted to equivalent plain Markdown. + +Source of truth: tutorials//_index.md (shortcodes, used by doc site) +Generated output: tutorials//README.md (plain Markdown, shown on GitHub) + +Usage: + python3 scripts/render-tutorials.py # write README.md files + python3 scripts/render-tutorials.py --check # fail if any are stale +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +TUTORIALS_DIR = REPO_ROOT / "tutorials" + +GENERATED_BANNER = ( + "\n\n" +) + +_ATTR_RE = re.compile(r'(\w+)\s*=\s*"([^"]*)"') + +_FRONT_MATTER_RE = re.compile(r"\A---\n.*?\n---\n", re.DOTALL) + +_TABPANE_RE = re.compile( + r"\{\{<\s*tabpane[^}]*>\}\}(?P.*?)\{\{<\s*/\s*tabpane\s*>\}\}", + re.DOTALL, +) +_CODE_TAB_RE = re.compile( + r"\{\{<\s*tab\s+(?P[^}]*?)>\}\}(?P.*?)\{\{<\s*/\s*tab\s*>\}\}", + re.DOTALL, +) +_MD_TAB_RE = re.compile( + r"\{\{%\s*tab\s+(?P[^}]*?)%\}\}(?P.*?)\{\{%\s*/\s*tab\s*%\}\}", + re.DOTALL, +) +_ALERT_RE = re.compile( + r"\{\{%\s*alert\s+(?P[^}]*?)%\}\}(?P.*?)\{\{%\s*/\s*alert\s*%\}\}", + re.DOTALL, +) +_CARD_GRID_RE = re.compile( + r'^\n(?P.*?)\n[ \t]*$', + re.DOTALL | re.MULTILINE, +) +_CARD_RE = re.compile( + r'[^"]+)">.*?' + r'unified-card-title">(?P.*?)</h3>.*?' + r'unified-card-summary">(?P<summary>.*?)</p>.*?</a>', + re.DOTALL, +) +_FLOW_DIAGRAM_RE = re.compile( + r'^<div\s+class="flow-diagram">\n(?P<body>.*?)\n</div>[ \t]*$', + re.DOTALL | re.MULTILINE, +) +_FLOW_STEP_RE = re.compile( + r'flow-step__label">(?P<label>.*?)</div>.*?' + r'flow-step__description">(?P<description>.*?)</div>', + re.DOTALL, +) + + +def _parse_attrs(raw: str) -> dict[str, str]: + return {m.group(1): m.group(2) for m in _ATTR_RE.finditer(raw)} + + +def _render_tabpane(match: re.Match[str]) -> str: + body = match.group("body") + + rendered: list[tuple[int, str]] = [] + + for m in _CODE_TAB_RE.finditer(body): + attrs = _parse_attrs(m.group("attrs")) + header = attrs.get("header", "").strip() + lang = attrs.get("lang", "").strip() + code = m.group("body").strip("\n") + block = f"**{header}**\n\n```{lang}\n{code}\n```" + rendered.append((m.start(), block)) + + for m in _MD_TAB_RE.finditer(body): + attrs = _parse_attrs(m.group("attrs")) + header = attrs.get("header", "").strip() + content = m.group("body").strip("\n") + block = f"**{header}**\n\n{content}" + rendered.append((m.start(), block)) + + rendered.sort(key=lambda item: item[0]) + return "\n\n".join(block for _, block in rendered) + + +def _render_alert(match: re.Match[str]) -> str: + attrs = _parse_attrs(match.group("attrs")) + title = attrs.get("title", "").strip() + body = match.group("body").strip("\n") + + lines: list[str] = [] + if title: + lines.append(f"> **{title}**") + lines.append(">") + for line in body.splitlines(): + lines.append(">" if not line.strip() else f"> {line}") + return "\n".join(lines) + + +def _render_card_grid(match: re.Match[str]) -> str: + """Convert a Docsy ``card-grid`` of link cards into a Markdown list. + + GitHub renders ``<a href><div>...</div></a>`` cards as empty links because + it hoists the block-level ``<div>`` out of the anchor. The doc site keeps + the cards; here we emit a plain bullet list of named links instead. + """ + items: list[str] = [] + for card in _CARD_RE.finditer(match.group("body")): + href = card.group("href").strip() + title = " ".join(card.group("title").split()) + summary = " ".join(card.group("summary").split()) + items.append(f"- **[{title}]({href})** — {summary}") + return "\n".join(items) if items else match.group(0) + + +def _render_flow_diagram(match: re.Match[str]) -> str: + """Convert a Docsy ``flow-diagram`` into a Markdown arrow chain plus list. + + GitHub renders the nested ``<div>`` steps as stray words, so we emit an + arrow chain of the step labels followed by a bullet list of each step's + description. The doc site keeps the styled diagram. + """ + steps = [ + (" ".join(m.group("label").split()), " ".join(m.group("description").split())) + for m in _FLOW_STEP_RE.finditer(match.group("body")) + ] + if not steps: + return match.group(0) + + chain = " → ".join(f"**{label}**" for label, _ in steps) + bullets = "\n".join(f"- **{label}** — {desc}" for label, desc in steps) + return f"{chain}\n\n{bullets}" + + +_HEADING_ID_RE = re.compile( + r"^(?P<hashes>#{1,6})\s+(?P<text>.*?)\s*\{#(?P<id>[\w-]+)\}\s*$", re.MULTILINE +) + + +def _github_slug(text: str) -> str: + """Approximate GitHub's heading-to-anchor slug algorithm.""" + slug = text.strip().lower() + # Drop characters GitHub strips (anything not a word char, space, or hyphen). + slug = re.sub(r"[^\w\s-]", "", slug) + # Collapse whitespace runs to single hyphens. + slug = re.sub(r"\s+", "-", slug) + return slug + + +def _rewrite_heading_ids(source: str) -> str: + """Strip Hugo ``{#id}`` heading attributes and repoint in-page links. + + Hugo uses ``## Title {#id}`` to set an explicit anchor, and links such as + ``[Title](#id)`` rely on it. Plain Markdown viewers render the ``{#id}`` + literally and instead auto-generate anchors from the heading text, so we + remove the attribute and rewrite every ``#id`` link to the slug GitHub + derives from the heading text. + """ + id_to_slug: dict[str, str] = {} + for match in _HEADING_ID_RE.finditer(source): + id_to_slug[match.group("id")] = _github_slug(match.group("text")) + + # Remove the {#id} attribute from headings. + out = _HEADING_ID_RE.sub(lambda m: f"{m.group('hashes')} {m.group('text')}", source) + + # Repoint in-page links that targeted the explicit ids. + def _replace_link(m: re.Match[str]) -> str: + slug = id_to_slug.get(m.group("id")) + return f"](#{slug})" if slug else m.group(0) + + out = re.sub(r"\]\(#(?P<id>[\w-]+)\)", _replace_link, out) + return out + + +def render(source: str) -> str: + out = _FRONT_MATTER_RE.sub("", source) + out = _rewrite_heading_ids(out) + out = _CARD_GRID_RE.sub(_render_card_grid, out) + out = _FLOW_DIAGRAM_RE.sub(_render_flow_diagram, out) + out = _TABPANE_RE.sub(_render_tabpane, out) + out = _ALERT_RE.sub(_render_alert, out) + # Collapse 3+ blank lines that conversions can introduce. + out = re.sub(r"\n{4,}", "\n\n\n", out) + return GENERATED_BANNER + out.lstrip("\n") + + +def iter_sources() -> list[Path]: + if not TUTORIALS_DIR.is_dir(): + return [] + return sorted(TUTORIALS_DIR.glob("*/_index.md")) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="Verify generated READMEs are up to date; do not write.", + ) + args = parser.parse_args() + + sources = iter_sources() + if not sources: + print("No tutorials/*/_index.md sources found.", file=sys.stderr) + return 0 + + stale: list[Path] = [] + for source in sources: + target = source.with_name("README.md") + expected = render(source.read_text(encoding="utf-8")) + current = target.read_text(encoding="utf-8") if target.exists() else None + + if args.check: + if current != expected: + stale.append(target) + continue + + if current != expected: + target.write_text(expected, encoding="utf-8") + print(f"rendered {target.relative_to(REPO_ROOT)}") + else: + print(f"unchanged {target.relative_to(REPO_ROOT)}") + + if args.check and stale: + print( + "The following generated READMEs are out of date:\n" + + "\n".join(f" - {p.relative_to(REPO_ROOT)}" for p in stale) + + "\n\nRun `python3 scripts/render-tutorials.py` and commit the result.", + file=sys.stderr, + ) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tutorials/README.md b/tutorials/README.md new file mode 100644 index 0000000..2b917a3 --- /dev/null +++ b/tutorials/README.md @@ -0,0 +1,25 @@ +# Tutorials + +Hands-on, end-to-end walkthroughs built on [`drasi-lib`](https://pypi.org/project/drasi-lib/), +the Python binding for Drasi's continuous-query engine. Each tutorial is a +self-contained project you can clone and run. + +| Tutorial | What you learn | How to run it | +| --- | --- | --- | +| [getting-started](getting-started) | The change-driven basics — PostgreSQL CDC and three continuous queries (a filter, a `count` aggregation, and a temporal `inactive-people`), printed live by a **Python reaction** in a no-UI console app you drive with SQL | Open the **Getting Started Tutorial (Python)** dev container (or a Codespace) and follow [tutorials/getting-started](getting-started) | +| [building-comfort](building-comfort) | Smart-building comfort monitoring — PostgreSQL CDC, six continuous queries with synthetic joins and aggregation, and a **Python reaction that drives a Streamlit UI** (with simulation, reset and per-room controls) | Open the **Building Comfort Tutorial (Python)** dev container (or a Codespace) and follow [tutorials/building-comfort](building-comfort) | +| [curbside-pickup](curbside-pickup) | Cross-database joins — a PostgreSQL `orders` store and a MySQL `vehicles` store joined by license plate, six continuous queries (incl. a temporal `drasi.trueFor` delay), rendered and driven from **one integrated Streamlit UI** powered by a Python reaction | Open the **Curbside Pickup Tutorial (Python)** dev container (or a Codespace) and follow [tutorials/curbside-pickup](curbside-pickup) | + +## How these tutorials are authored + +Each tutorial is written once in `tutorials/<name>/_index.md` (the single source +of truth), which is mounted into the [documentation site](../website) so it also +renders at <https://drasi-project.github.io/drasi-python/docs/tutorials/>. The +GitHub-friendly `tutorials/<name>/README.md` is generated from that same file: + +```bash +python3 scripts/render-tutorials.py # regenerate the READMEs +python3 scripts/render-tutorials.py --check # fail if any are stale (run in CI) +``` + +Edit the `_index.md`, not the `README.md`. diff --git a/tutorials/building-comfort/.env.example b/tutorials/building-comfort/.env.example new file mode 100644 index 0000000..aa6ed00 --- /dev/null +++ b/tutorials/building-comfort/.env.example @@ -0,0 +1,23 @@ +# Example environment variables for the building-comfort tutorial. +# +# Copy this file to .env and adjust as needed: +# cp .env.example .env +# +# The scripts and the app read these with the same defaults shown here, so you +# only need a .env if you change a port or credential. +# +# IMPORTANT: Never commit .env with real credentials! + +# PostgreSQL connection settings (used by the app via psycopg and by Drasi). +POSTGRES_HOST=localhost +POSTGRES_PORT=5732 +POSTGRES_HOST_PORT=5732 +POSTGRES_DATABASE=building_comfort +POSTGRES_USER=drasi_user +POSTGRES_PASSWORD=drasi_password + +# Streamlit UI port. +STREAMLIT_PORT=8501 + +# How often simulation mode assigns a random room new readings, in seconds. +SIMULATION_INTERVAL_S=3 diff --git a/tutorials/building-comfort/.gitignore b/tutorials/building-comfort/.gitignore new file mode 100644 index 0000000..1e58996 --- /dev/null +++ b/tutorials/building-comfort/.gitignore @@ -0,0 +1,11 @@ +# Local environment overrides +.env + +# Python +__pycache__/ +*.pyc +.venv/ + +# Drasi engine state / logs +data/ +*.log diff --git a/tutorials/building-comfort/.streamlit/config.toml b/tutorials/building-comfort/.streamlit/config.toml new file mode 100644 index 0000000..b1b7854 --- /dev/null +++ b/tutorials/building-comfort/.streamlit/config.toml @@ -0,0 +1,8 @@ +# Streamlit settings for the tutorial. `headless = true` skips the interactive +# first-run email prompt (which would otherwise block startup and leave the port +# unbound), and `address = 0.0.0.0` makes the app reachable through a dev +# container's / Codespaces' forwarded port. +[server] +headless = true +address = "0.0.0.0" +port = 8501 diff --git a/tutorials/building-comfort/README.md b/tutorials/building-comfort/README.md new file mode 100644 index 0000000..c0288f0 --- /dev/null +++ b/tutorials/building-comfort/README.md @@ -0,0 +1,519 @@ +<!-- DO NOT EDIT. Generated from _index.md by scripts/render-tutorials.py. Edit _index.md and run `python3 scripts/render-tutorials.py`. --> + +Imagine you manage a building and want to know — the instant it happens — when any room becomes uncomfortable: too hot, too cold, or stuffy with CO2. You don't want to poll sensors on a timer or wire up a stream processor by hand. You just want to describe what "uncomfortable" means and have something watch every room, floor, and the whole building for you, continuously. + +This tutorial builds a **Building Comfort** monitoring demo with **`drasi-lib`**, the Python binding that embeds Drasi's continuous-query engine directly in your app. A PostgreSQL database holds a building made of floors and rooms, each room reporting `temperature`, `humidity`, and `co2`. You'll run a single Streamlit program that starts the engine, watch a live UI light up, and then change room readings and see everything react in real time. Instead of a built-in dashboard reaction, the UI is rendered from a **Python reaction** — a callback that receives each query's changes and updates the screen. + +**What you'll build:** a Streamlit app that embeds Drasi, connects to PostgreSQL, and reacts to room sensor changes in real time, assembled from Drasi's three core building blocks: + +**Sources** → **Continuous Queries** → **Reactions** + +- **Sources** — Connect to your data sources +- **Continuous Queries** — Define what changes matter +- **Reactions** — Take action automatically + +| Step | What You'll Do | Time | +| ---- | ------------- | ---- | +| **[Step 1: Set Up Your Environment](#step-1-of-4-set-up-your-environment)** | Open the dev container (or install Python + Docker locally) | 5 min | +| **[Step 2: Run the Demo](#step-2-of-4-run-the-demo)** | One command starts PostgreSQL and the Streamlit app | 3 min | +| **[Step 3: Open the UI](#step-3-of-4-open-the-ui)** | Watch comfort levels and alerts live | 2 min | +| **[Step 4: Drive Change](#step-4-of-4-drive-change)** | Set, reset, and simulate rooms from the sidebar — and watch Drasi react instantly | 5 min | +| **[How It Works](#how-it-works)** | Understand the source, the six queries, the synthetic joins, and the Python reaction | 5 min | + +> **Before you begin** +> +> - **One process:** the Streamlit app embeds the Drasi engine, so there's just one command to run. It stays in the foreground. +> - **Working directory:** run every command from the tutorial directory (`tutorials/building-comfort/`). The dev container opens there automatically; if you're running locally, `cd tutorials/building-comfort` first. +> - **Command tabs:** commands are shown in tabs (*bash / zsh* and *PowerShell*) — use the one for your shell. The dev container and Codespaces use *bash*. +> - **Ports:** the Streamlit UI is on `8501` and PostgreSQL is published on `5732`. + +## Step 1 of 4: Set Up Your Environment +The easiest way to follow this tutorial is the **dev container**, which installs everything for you. You can also run locally if you prefer. + +### Option A: Dev Container or GitHub Codespaces (recommended) + +1. Open the [`drasi-python`](https://github.com/drasi-project/drasi-python) repository in VS Code and run **Reopen in Container** (or create a **Codespace** from the repo's **Code** menu). +2. When prompted for a configuration, choose **Building Comfort Tutorial (Python)**. +3. Wait for the container to finish. Its setup script installs the tutorial's Python dependencies (`drasi-lib`, `streamlit`, `psycopg`) and the Docker tooling for PostgreSQL. + +That's it — skip ahead to [Step 2](#step-2-of-4-run-the-demo). + +### Option B: Run Locally + +You'll need **Python 3.10+**, **Docker** (for PostgreSQL), and **bash** for the helper scripts (on Windows use Git Bash or WSL, or run the PowerShell scripts). From the repository root, move into the tutorial directory and install the dependencies: + +**bash / zsh** + +```bash +cd tutorials/building-comfort +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +``` + +**PowerShell** + +```powershell +cd tutorials/building-comfort +python -m venv .venv; .venv\Scripts\Activate.ps1 +pip install -r requirements.txt +``` + +`drasi-lib` ships as a prebuilt wheel, so no Rust toolchain is needed. It downloads the Postgres plugins it needs from `ghcr.io/drasi-project` the first time the app runs. + +## Step 2 of 4: Run the Demo +In your terminal, start the demo: + +**bash / zsh** + +```bash +bash scripts/start-demo.sh +``` + +**PowerShell** + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/start-demo.ps1 +``` + +The `start-demo` script does two things: it starts PostgreSQL (seeding one building, three floors, and nine rooms — every room comfortable to begin with) and then launches the Streamlit app. + +On first start, the app's embedded engine downloads the plugins it needs (`source/postgres`, `bootstrap/postgres`) from `ghcr.io/drasi-project` and caches them under `~/.drasi/plugins`, connects to the database, and starts the six continuous queries and the reaction. Streamlit prints a URL like the following when it's ready: + +```text +You can now view your Streamlit app in your browser. +Local URL: http://localhost:8501 +``` + +Leave this running. Everything else happens in your browser. + +> **Stopping and resetting** +> +> Press **Ctrl+C** in the terminal to stop the app. To remove the database container when you're completely done, run `bash scripts/cleanup.sh` (bash) or `powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1` (PowerShell). Add `--volumes` (bash) or `-RemoveVolumes` (PowerShell) to also delete the data. + +## Step 3 of 4: Open the UI +The app renders the building from the Python reaction — there's no separate dashboard server. **Wait until the app has finished starting** (on the first run this takes ~30 seconds while the plugins download and the queries bootstrap), then open it in your browser: + +```text +http://localhost:8501 +``` + +In the dev container or Codespaces, port `8501` is forwarded automatically — VS Code shows a notification when the app is ready, and you can also open it from the **Ports** panel. + +You'll see the **Building Comfort** UI: + +- **Building Comfort** (the main panel) — the building grouped by floor, with each room's comfort level and sensor readings. Every room starts at comfort `46` (comfortable). +- **Overall** — the building's overall comfort level as a number and a meter. +- **Comfort alerts** and **Floor alerts** — empty for now; rooms and floors appear here only when they need attention. +- **Floor comfort** — a table of each floor's comfort level. + +The UI refreshes on its own — the reaction updates a shared snapshot the moment the data changes, and Streamlit renders it. Let's make something change. + +## Step 4 of 4: Drive Change +Everything you need is in the **sidebar** on the left. Use it to change room readings and watch Drasi react. + +> **No middle tier — the controls just write to the database** +> +> The sidebar controls don't call the Drasi engine directly. Each one runs a plain SQL `UPDATE` against PostgreSQL (through `psycopg`) — exactly what an existing building-management app would already do. There's no API to call and no event to publish. Drasi observes the row change through PostgreSQL's logical replication (CDC), re-evaluates the affected queries, and calls the Python reaction, which updates the UI. The set/reset/simulate buttons are a thin front end over a single `UPDATE` statement. + +### Set a room + +In the sidebar's **Set a room** section, pick a room and give it new readings. Try pushing one out of the comfortable band — for example `temperature = 40`, `humidity = 20`, `co2 = 700` — then press **Set room**. + +Within about a second the UI reacts: that room drops out of the comfortable band, the overall meter falls, and entries appear in **Comfort alerts** and **Floor alerts**. + +### Reset a room (or all rooms) + +Press **Reset this room** to return the selected room to the comfortable defaults (`70 / 40 / 10`), or **Reset all rooms** to return the whole building. The alerts clear and the building returns to green. + +### Set custom values + +Try partial degradation — make a room too hot without touching CO2. Pick a room and set `temperature = 82`, `humidity = 40`, `co2 = 10`: + +That gives `50 + (82-72) + (40-42) + 0 = 58` — above 50, so the room and its floor raise alerts even though humidity and CO2 are fine. + +### Let it run hands-free + +Flip the **Simulation mode** toggle at the top of the sidebar. Every few seconds it picks a random room and assigns new readings, so comfort levels rise and fall and alerts come and go on their own. Flip it off to stop, then **Reset all rooms** to return everything to comfortable. + +## How It Works +The demo is two files in `tutorials/building-comfort/`: **`engine.py`** holds the source, the six queries, and the `ComfortEngine` (its Python reaction and the SQL writes); **`app.py`** renders the reaction's state and hosts the sidebar controls. The [complete `engine.py`](#putting-it-all-together) is at the end. Here's what each part does. + +### The Source + +The engine installs the Postgres plugins and adds the source, pointing it at the tutorial database: + +```python +await drasi.install_plugin("source/postgres") +await drasi.install_plugin("bootstrap/postgres") +await drasi.start() + +await drasi.add_source( + "postgres", + "db", + { + **POSTGRES, # host, port, user, password, database + "sslMode": "prefer", + # Schema-qualified so the node labels match the queries. + "tables": ["public.Building", "public.Floor", "public.Room"], + "tableKeys": [ + {"table": "Building", "keyColumns": ["id"]}, + {"table": "Floor", "keyColumns": ["id"]}, + {"table": "Room", "keyColumns": ["id"]}, + ], + "slotName": "drasi_building_comfort_slot", + "publicationName": "drasi_building_comfort_pub", + }, + bootstrap={"kind": "postgres", **POSTGRES}, +) +``` + +The source uses **logical replication (CDC)** to stream changes from the `Building`, `Floor`, and `Room` tables. `tableKeys` tells Drasi the primary key of each table so it can track row identity across changes. The `bootstrap` provider loads the rows that already exist when the query starts; after that, every `UPDATE` the UI makes flows to Drasi as a change. The table names are PascalCase so the node labels Drasi sees match the queries exactly: `(r:Room)`, `(f:Floor)`, `(b:Building)`. For every option the source accepts, see [Configure the PostgreSQL Source](https://drasi.io/drasi-server/how-to-guides/configuration/configure-sources/configure-postgresql-source/). + +### The Continuous Queries + +Each query computes a **comfort level** with the same formula. A value between **40 and 50** is comfortable; the seed values (70°F, 40%, 10 ppm) give `50 + (70-72) + (40-42) + 0 = 46`. + +```cypher +floor( 50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END ) +``` + +There are six queries: + +| Query | What it returns | +| ----- | --------------- | +| `building-comfort-ui` | One row per room with its comfort level — the feed that drives the building view | +| `building-comfort-level-calc` | The building's overall comfort level | +| `floor-comfort-level-calc` | Each floor's average comfort level | +| `room-alert` | Only the rooms whose comfort is outside 40–50 | +| `floor-alert` | Only the floors whose average comfort is outside 40–50 | +| `building-alert` | The building, when its overall comfort is outside 40–50 | + +#### Synthetic joins connect the entities + +PostgreSQL knows `Room.floor_id` references `Floor.id` through a foreign key, but Drasi doesn't read foreign keys. Instead, each query **declares** the relationships it needs as synthetic joins, so the Cypher can walk from room to floor to building. In `drasi-lib` a join is a plain mapping passed to `add_query`: + +```python +PART_OF_FLOOR = { + "id": "PART_OF_FLOOR", + "keys": [ + {"label": "Room", "property": "floor_id"}, + {"label": "Floor", "property": "id"}, + ], +} +PART_OF_BUILDING = { + "id": "PART_OF_BUILDING", + "keys": [ + {"label": "Floor", "property": "building_id"}, + {"label": "Building", "property": "id"}, + ], +} + +await drasi.add_query( + "building-comfort-ui", CYPHER, ["db"], joins=[PART_OF_FLOOR, PART_OF_BUILDING] +) +``` + +With those joins declared, a query can match the whole hierarchy: + +```cypher +MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor)-[:PART_OF_BUILDING]->(b:Building) +``` + +#### Aggregating up the hierarchy + +Half of the queries don't just read rooms — they **roll comfort levels up** the building. In a Continuous Query, an aggregate such as `avg()` inside a `WITH` groups by whatever non-aggregated values travel alongside it, exactly like SQL's `GROUP BY`. + +**One stage — average a floor's rooms.** `floor-comfort-level-calc` keeps the floor `f` in the `WITH`, so `avg()` produces one value per floor: + +```cypher +WITH f, floor( 50 + (r.temperature - 72) + ... ) AS RoomComfortLevel +WITH f, avg(RoomComfortLevel) AS ComfortLevel +RETURN f.id AS FloorId, ComfortLevel +``` + +**Two stages — average the averages.** `building-alert` aggregates twice. Carrying `f, b` groups the first `avg()` by floor; dropping `f` from the next `WITH` widens the grouping so the second `avg()` rolls the floor averages up into a single building level: + +```cypher +WITH f, b, avg(RoomComfortLevel) AS FloorComfortLevel // rooms -> floor +WITH b, avg(FloorComfortLevel) AS ComfortLevel // floors -> building +``` + +**Filtering on an aggregate.** The `floor-alert` and `building-alert` queries place a `WHERE` *after* the aggregation — like SQL's `HAVING` — so a floor or the building only shows up while its average is outside the comfortable band: + +```cypher +WITH f, avg(RoomComfortLevel) AS ComfortLevel +WHERE ComfortLevel < 40 OR ComfortLevel > 50 +RETURN f.id AS FloorId, ComfortLevel +``` + +Because these are *continuous* queries, the aggregates are maintained **incrementally**: when one room's reading changes, Drasi recomputes only the affected floor and building averages and emits just that change — it never rescans every room. + +### The Python Reaction + +This is where the Python demo differs from a built-in dashboard. Instead of configuring a dashboard reaction, we register a **Python reaction** — a plain callback subscribed to all six queries. Each time a query's result set changes, Drasi calls it with the query id and the diffs, and the callback updates a thread-safe snapshot: + +```python +def on_results(event): + query = QUERIES_BY_ID[event["query_id"]] + with lock: + store = snapshot[query.id] # this query's rows, keyed by id; the UI reads it + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[query.key], None) + case {"after": dict() as row} | { + "data": dict() as row + }: # ADD / UPDATE / aggregation + store[row[query.key]] = row + + +await drasi.add_python_reaction("ui", [q.id for q in QUERIES], on_results) +``` + +Each query knows the RETURN column that identifies its rows (`query.key` — `RoomId`, `FloorId` or `BuildingId`). Every diff carries the affected row in `data` (for an `UPDATE` it equals `after`), so the snapshot is kept by row identity — upsert on `ADD`/`UPDATE`, drop on `DELETE`. The `type` is one of `ADD`, `UPDATE`, or `DELETE`. This works the same for the per-room queries and the aggregate floor/building queries, which emit an `UPDATE` as their average changes. See [Python reactions](../../guides/python-reactions/) for the full callback contract. + +### Embedding the engine in Streamlit + +Streamlit re-runs its script top to bottom on every interaction, which doesn't suit a long-lived async engine. So the engine runs on its own asyncio event loop in a **background daemon thread**, created exactly once with `@st.cache_resource` and shared across every browser session: + +```python +@st.cache_resource +def get_engine(): + engine = ComfortEngine() # spins up the background loop + Drasi + engine.wait_ready() + return engine +``` + +Each run, `app.py` reads `engine.snapshot()` and renders the building, the meter, the floor table and the alert lists. A one-and-a-half-second auto-refresh keeps the view live. Because the reaction only updates the snapshot when a query actually changes, the UI reflects each `UPDATE` the moment it lands — no polling of the database. + +The centerpiece building panel is just a loop over the `building-comfort-ui` rows, grouped by floor, picking a status emoji from each room's comfort level: + +```python +for floor_name in sorted(by_floor): + st.markdown(f"### {floor_name}") + for room in by_floor[floor_name]: + emoji, label = comfort_status(room["ComfortLevel"]) + st.markdown( + f"- **{room['RoomName']}** — comfort **{room['ComfortLevel']}** {emoji} {label}" + ) +``` + +And the reaction renders it live: + +<img src="images/streamlit-building-view.png" width="820" alt="The Building Comfort UI: rooms grouped by floor, each tagged with a comfort level and a green, red, or blue status emoji, with a sidebar of controls"> + +### Other ways to consume a query + +A Python reaction is the most direct way to turn query changes into a UI, but it isn't the only option `drasi-lib` gives you: + +- **Stream the changes** with `async for event in await drasi.query_results("room-alert")` and drive any front end you like. See [Streaming results](../../guides/streaming/). +- **Reuse a published reaction** such as the dashboard reaction plugin, exactly as the [Postgres dashboard example](../../examples/postgres-dashboard/) does — no UI code at all. + +Because Drasi emits only what *changed* — never the full result set — whichever you choose stays live without polling. + +## Putting It All Together +The sections above walked through the Drasi side one piece at a time. Here it is assembled into the single `engine.py` — the source, the two synthetic joins, the six queries, and the `ComfortEngine` that runs them and writes back. `app.py` (the Streamlit UI) imports `ComfortEngine`, the query ids, and the comfort band from it: + +```python +import asyncio +import os +import random +import threading +from dataclasses import dataclass, field + +os.environ.setdefault("RUST_LOG", "warn") # quiet the engine's INFO logging + +import psycopg +from drasi import Drasi + +# --- The source: the tutorial's PostgreSQL database --- +CONNECTION = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5732")), + "database": os.environ.get("POSTGRES_DATABASE", "building_comfort"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), + "sslMode": "prefer", +} +SOURCE_CONFIG = { + **CONNECTION, + "tables": ["public.Building", "public.Floor", "public.Room"], + "tableKeys": [ + {"table": "Building", "keyColumns": ["id"]}, + {"table": "Floor", "keyColumns": ["id"]}, + {"table": "Room", "keyColumns": ["id"]}, + ], + "slotName": "drasi_building_comfort_slot", + "publicationName": "drasi_building_comfort_pub", +} +BOOTSTRAP_CONFIG = {"kind": "postgres", **CONNECTION} +# The UI writes room updates through a psycopg connection built from CONNECTION +# (psycopg spells the database key "dbname"), so Drasi observes them via CDC. + +COMFORT_DEFAULTS = {"temperature": 70, "humidity": 40, "co2": 10} +COMFORT_MIN, COMFORT_MAX = 40, 50 + +# --- Two synthetic joins walk Room -> Floor -> Building --- +PART_OF_FLOOR = { + "id": "PART_OF_FLOOR", + "keys": [ + {"label": "Room", "property": "floor_id"}, + {"label": "Floor", "property": "id"}, + ], +} +PART_OF_BUILDING = { + "id": "PART_OF_BUILDING", + "keys": [ + {"label": "Floor", "property": "building_id"}, + {"label": "Building", "property": "id"}, + ], +} + + +@dataclass(frozen=True) +class Query: + id: str + key: str # the RETURN column that identifies each row + cypher: str + joins: list[dict] = field(default_factory=list) + + +# Every query computes the same comfort level; a value in 40-50 is comfortable. +QUERIES = [ + # building-comfort-ui: one row per room with its comfort level. Drives the + # building view; walks Room -> Floor -> Building through the two joins. + Query( + id="building-comfort-ui", + key="RoomId", + joins=[PART_OF_FLOOR, PART_OF_BUILDING], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor)-[:PART_OF_BUILDING]->(b:Building) + RETURN r.id AS RoomId, r.name AS RoomName, f.id AS FloorId, f.name AS FloorName, + b.name AS BuildingName, r.temperature AS Temperature, r.humidity AS Humidity, + r.co2 AS CO2, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS ComfortLevel + """, + ), + # floor-comfort-level-calc: each floor's comfort, the average of its rooms. + Query( + id="floor-comfort-level-calc", + key="FloorId", + joins=[PART_OF_FLOOR], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor) + WITH f, floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS RoomComfort + WITH f, avg(RoomComfort) AS ComfortLevel + RETURN f.id AS FloorId, ComfortLevel + """, + ), + # room-alert: only the rooms whose comfort is outside the 40-50 band. + Query( + id="room-alert", + key="RoomId", + cypher=""" + MATCH (r:Room) + WITH r.id AS RoomId, r.name AS RoomName, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS ComfortLevel + WHERE ComfortLevel < 40 OR ComfortLevel > 50 + RETURN RoomId, RoomName, ComfortLevel + """, + ), + # building-comfort-level-calc (average of floor averages), floor-alert, and + # building-alert follow the same formula with avg()/WHERE variations. +] + + +class ComfortEngine: + """Runs Drasi on a background loop and exposes a thread-safe snapshot.""" + + def __init__(self): + self._lock = threading.Lock() + self._results = {} # query id -> {row key -> row} + self._key_by_id = {q.id: q.key for q in QUERIES} + self._simulation = False + self._ready = threading.Event() + self._drasi = None + threading.Thread(target=self._run, daemon=True).start() + + def _run(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._start()) + self._ready.set() + loop.run_forever() + + async def _start(self): + self._drasi = await Drasi.create("building-comfort") + for plugin in ("source/postgres", "bootstrap/postgres"): + await self._drasi.install_plugin(plugin) + await self._drasi.start() + + await self._drasi.add_source("postgres", "db", SOURCE_CONFIG, bootstrap=BOOTSTRAP_CONFIG) + for q in QUERIES: + await self._drasi.add_query(q.id, q.cypher, ["db"], joins=q.joins or None) + for q in QUERIES: + await self._drasi.wait_for_query(q.id) + rows = await self._drasi.get_query_results(q.id) + self._results[q.id] = {r[q.key]: r for r in rows} + + await self._drasi.add_python_reaction("ui", [q.id for q in QUERIES], self._on_results) + + def _on_results(self, event): + key = self._key_by_id[event["query_id"]] + with self._lock: + store = self._results.setdefault(event["query_id"], {}) + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[key], None) + case {"after": dict() as row} | {"data": dict() as row}: + store[row[key]] = row + + def snapshot(self): + with self._lock: + return { + "simulation": self._simulation, + "results": {q: list(s.values()) for q, s in self._results.items()}, + } + + # set_room(id, t, h, co2) and reset_room(id=None) write UPDATEs to Postgres; + # set_simulation(on) toggles a background task that assigns a random room new + # readings every few seconds. Drasi observes each write via CDC. +``` + +That's the whole Drasi side in one file: one source, two synthetic joins up the Room -> Floor -> Building hierarchy, six continuous queries (four of them aggregations), and a reaction that keeps a snapshot the UI renders and drives. + +## Clean Up +When you're finished, stop the app with **Ctrl+C** in the terminal, then remove the database container: + +**bash / zsh** + +```bash +# Stop containers, keep data +bash scripts/cleanup.sh + +# Stop containers and delete the data volume +bash scripts/cleanup.sh --volumes +``` + +**PowerShell** + +```powershell +# Stop containers, keep data +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 + +# Stop containers and delete the data volume +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 -RemoveVolumes +``` + +## What You Learned +- **Sources** connect Drasi to live data — here, PostgreSQL via Change Data Capture, configured in a few lines of Python. +- **Continuous Queries** with **synthetic joins** let you model relationships (room → floor → building) and compute derived values (comfort levels) that stay current automatically. +- **Reactions** turn query changes into action. A **Python reaction** gave you a live UI from a callback and a snapshot — no bespoke web server, and complete control over the rendering. +- Because Drasi emits only what *changed*, everything updates the instant the data does, with no polling. + +From here, try editing the comfort formula or the layout in `app.py`, adding a new alert query, or pointing the source at your own database. diff --git a/tutorials/building-comfort/_index.md b/tutorials/building-comfort/_index.md new file mode 100644 index 0000000..a70d4aa --- /dev/null +++ b/tutorials/building-comfort/_index.md @@ -0,0 +1,553 @@ +--- +type: "docs" +title: "Building Comfort" +linkTitle: "Building Comfort" +weight: 20 +description: > + Monitor a smart building's comfort in real time with PostgreSQL CDC, six continuous queries with synthetic joins, and a Python reaction that drives a Streamlit UI. +--- + +Imagine you manage a building and want to know — the instant it happens — when any room becomes uncomfortable: too hot, too cold, or stuffy with CO2. You don't want to poll sensors on a timer or wire up a stream processor by hand. You just want to describe what "uncomfortable" means and have something watch every room, floor, and the whole building for you, continuously. + +This tutorial builds a **Building Comfort** monitoring demo with **`drasi-lib`**, the Python binding that embeds Drasi's continuous-query engine directly in your app. A PostgreSQL database holds a building made of floors and rooms, each room reporting `temperature`, `humidity`, and `co2`. You'll run a single Streamlit program that starts the engine, watch a live UI light up, and then change room readings and see everything react in real time. Instead of a built-in dashboard reaction, the UI is rendered from a **Python reaction** — a callback that receives each query's changes and updates the screen. + +**What you'll build:** a Streamlit app that embeds Drasi, connects to PostgreSQL, and reacts to room sensor changes in real time, assembled from Drasi's three core building blocks: + +<div class="flow-diagram"> + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-database"></i> + </div> + <div class="flow-step__label">Sources</div> + <div class="flow-step__description">Connect to your data sources</div> + </div> + + <div class="flow-arrow"> + <i class="fas fa-arrow-right"></i> + </div> + + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-filter"></i> + </div> + <div class="flow-step__label">Continuous Queries</div> + <div class="flow-step__description">Define what changes matter</div> + </div> + + <div class="flow-arrow"> + <i class="fas fa-arrow-right"></i> + </div> + + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-bolt"></i> + </div> + <div class="flow-step__label">Reactions</div> + <div class="flow-step__description">Take action automatically</div> + </div> +</div> + +| Step | What You'll Do | Time | +| ---- | ------------- | ---- | +| **[Step 1: Set Up Your Environment](#setup)** | Open the dev container (or install Python + Docker locally) | 5 min | +| **[Step 2: Run the Demo](#run)** | One command starts PostgreSQL and the Streamlit app | 3 min | +| **[Step 3: Open the UI](#ui)** | Watch comfort levels and alerts live | 2 min | +| **[Step 4: Drive Change](#drive)** | Set, reset, and simulate rooms from the sidebar — and watch Drasi react instantly | 5 min | +| **[How It Works](#how)** | Understand the source, the six queries, the synthetic joins, and the Python reaction | 5 min | + +{{% alert title="Before you begin" color="info" %}} +- **One process:** the Streamlit app embeds the Drasi engine, so there's just one command to run. It stays in the foreground. +- **Working directory:** run every command from the tutorial directory (`tutorials/building-comfort/`). The dev container opens there automatically; if you're running locally, `cd tutorials/building-comfort` first. +- **Command tabs:** commands are shown in tabs (*bash / zsh* and *PowerShell*) — use the one for your shell. The dev container and Codespaces use *bash*. +- **Ports:** the Streamlit UI is on `8501` and PostgreSQL is published on `5732`. +{{% /alert %}} + +## Step 1 of 4: Set Up Your Environment {#setup} + +The easiest way to follow this tutorial is the **dev container**, which installs everything for you. You can also run locally if you prefer. + +### Option A: Dev Container or GitHub Codespaces (recommended) + +1. Open the [`drasi-python`](https://github.com/drasi-project/drasi-python) repository in VS Code and run **Reopen in Container** (or create a **Codespace** from the repo's **Code** menu). +2. When prompted for a configuration, choose **Building Comfort Tutorial (Python)**. +3. Wait for the container to finish. Its setup script installs the tutorial's Python dependencies (`drasi-lib`, `streamlit`, `psycopg`) and the Docker tooling for PostgreSQL. + +That's it — skip ahead to [Step 2](#run). + +### Option B: Run Locally + +You'll need **Python 3.10+**, **Docker** (for PostgreSQL), and **bash** for the helper scripts (on Windows use Git Bash or WSL, or run the PowerShell scripts). From the repository root, move into the tutorial directory and install the dependencies: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +cd tutorials/building-comfort +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +cd tutorials/building-comfort +python -m venv .venv; .venv\Scripts\Activate.ps1 +pip install -r requirements.txt +{{< /tab >}} +{{< /tabpane >}} + +`drasi-lib` ships as a prebuilt wheel, so no Rust toolchain is needed. It downloads the Postgres plugins it needs from `ghcr.io/drasi-project` the first time the app runs. + +## Step 2 of 4: Run the Demo {#run} + +In your terminal, start the demo: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +bash scripts/start-demo.sh +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +powershell -ExecutionPolicy Bypass -File scripts/start-demo.ps1 +{{< /tab >}} +{{< /tabpane >}} + +The `start-demo` script does two things: it starts PostgreSQL (seeding one building, three floors, and nine rooms — every room comfortable to begin with) and then launches the Streamlit app. + +On first start, the app's embedded engine downloads the plugins it needs (`source/postgres`, `bootstrap/postgres`) from `ghcr.io/drasi-project` and caches them under `~/.drasi/plugins`, connects to the database, and starts the six continuous queries and the reaction. Streamlit prints a URL like the following when it's ready: + +```text +You can now view your Streamlit app in your browser. +Local URL: http://localhost:8501 +``` + +Leave this running. Everything else happens in your browser. + +{{% alert title="Stopping and resetting" color="info" %}} +Press **Ctrl+C** in the terminal to stop the app. To remove the database container when you're completely done, run `bash scripts/cleanup.sh` (bash) or `powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1` (PowerShell). Add `--volumes` (bash) or `-RemoveVolumes` (PowerShell) to also delete the data. +{{% /alert %}} + +## Step 3 of 4: Open the UI {#ui} + +The app renders the building from the Python reaction — there's no separate dashboard server. **Wait until the app has finished starting** (on the first run this takes ~30 seconds while the plugins download and the queries bootstrap), then open it in your browser: + +```text +http://localhost:8501 +``` + +In the dev container or Codespaces, port `8501` is forwarded automatically — VS Code shows a notification when the app is ready, and you can also open it from the **Ports** panel. + +You'll see the **Building Comfort** UI: + +- **Building Comfort** (the main panel) — the building grouped by floor, with each room's comfort level and sensor readings. Every room starts at comfort `46` (comfortable). +- **Overall** — the building's overall comfort level as a number and a meter. +- **Comfort alerts** and **Floor alerts** — empty for now; rooms and floors appear here only when they need attention. +- **Floor comfort** — a table of each floor's comfort level. + +The UI refreshes on its own — the reaction updates a shared snapshot the moment the data changes, and Streamlit renders it. Let's make something change. + +## Step 4 of 4: Drive Change {#drive} + +Everything you need is in the **sidebar** on the left. Use it to change room readings and watch Drasi react. + +{{% alert title="No middle tier — the controls just write to the database" color="info" %}} +The sidebar controls don't call the Drasi engine directly. Each one runs a plain SQL `UPDATE` against PostgreSQL (through `psycopg`) — exactly what an existing building-management app would already do. There's no API to call and no event to publish. Drasi observes the row change through PostgreSQL's logical replication (CDC), re-evaluates the affected queries, and calls the Python reaction, which updates the UI. The set/reset/simulate buttons are a thin front end over a single `UPDATE` statement. +{{% /alert %}} + +### Set a room + +In the sidebar's **Set a room** section, pick a room and give it new readings. Try pushing one out of the comfortable band — for example `temperature = 40`, `humidity = 20`, `co2 = 700` — then press **Set room**. + +Within about a second the UI reacts: that room drops out of the comfortable band, the overall meter falls, and entries appear in **Comfort alerts** and **Floor alerts**. + +### Reset a room (or all rooms) + +Press **Reset this room** to return the selected room to the comfortable defaults (`70 / 40 / 10`), or **Reset all rooms** to return the whole building. The alerts clear and the building returns to green. + +### Set custom values + +Try partial degradation — make a room too hot without touching CO2. Pick a room and set `temperature = 82`, `humidity = 40`, `co2 = 10`: + +That gives `50 + (82-72) + (40-42) + 0 = 58` — above 50, so the room and its floor raise alerts even though humidity and CO2 are fine. + +### Let it run hands-free + +Flip the **Simulation mode** toggle at the top of the sidebar. Every few seconds it picks a random room and assigns new readings, so comfort levels rise and fall and alerts come and go on their own. Flip it off to stop, then **Reset all rooms** to return everything to comfortable. + +## How It Works {#how} + +The demo is two files in `tutorials/building-comfort/`: **`engine.py`** holds the source, the six queries, and the `ComfortEngine` (its Python reaction and the SQL writes); **`app.py`** renders the reaction's state and hosts the sidebar controls. The [complete `engine.py`](#all) is at the end. Here's what each part does. + +### The Source + +The engine installs the Postgres plugins and adds the source, pointing it at the tutorial database: + +```python +await drasi.install_plugin("source/postgres") +await drasi.install_plugin("bootstrap/postgres") +await drasi.start() + +await drasi.add_source( + "postgres", + "db", + { + **POSTGRES, # host, port, user, password, database + "sslMode": "prefer", + # Schema-qualified so the node labels match the queries. + "tables": ["public.Building", "public.Floor", "public.Room"], + "tableKeys": [ + {"table": "Building", "keyColumns": ["id"]}, + {"table": "Floor", "keyColumns": ["id"]}, + {"table": "Room", "keyColumns": ["id"]}, + ], + "slotName": "drasi_building_comfort_slot", + "publicationName": "drasi_building_comfort_pub", + }, + bootstrap={"kind": "postgres", **POSTGRES}, +) +``` + +The source uses **logical replication (CDC)** to stream changes from the `Building`, `Floor`, and `Room` tables. `tableKeys` tells Drasi the primary key of each table so it can track row identity across changes. The `bootstrap` provider loads the rows that already exist when the query starts; after that, every `UPDATE` the UI makes flows to Drasi as a change. The table names are PascalCase so the node labels Drasi sees match the queries exactly: `(r:Room)`, `(f:Floor)`, `(b:Building)`. For every option the source accepts, see [Configure the PostgreSQL Source](https://drasi.io/drasi-server/how-to-guides/configuration/configure-sources/configure-postgresql-source/). + +### The Continuous Queries + +Each query computes a **comfort level** with the same formula. A value between **40 and 50** is comfortable; the seed values (70°F, 40%, 10 ppm) give `50 + (70-72) + (40-42) + 0 = 46`. + +```cypher +floor( 50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END ) +``` + +There are six queries: + +| Query | What it returns | +| ----- | --------------- | +| `building-comfort-ui` | One row per room with its comfort level — the feed that drives the building view | +| `building-comfort-level-calc` | The building's overall comfort level | +| `floor-comfort-level-calc` | Each floor's average comfort level | +| `room-alert` | Only the rooms whose comfort is outside 40–50 | +| `floor-alert` | Only the floors whose average comfort is outside 40–50 | +| `building-alert` | The building, when its overall comfort is outside 40–50 | + +#### Synthetic joins connect the entities + +PostgreSQL knows `Room.floor_id` references `Floor.id` through a foreign key, but Drasi doesn't read foreign keys. Instead, each query **declares** the relationships it needs as synthetic joins, so the Cypher can walk from room to floor to building. In `drasi-lib` a join is a plain mapping passed to `add_query`: + +```python +PART_OF_FLOOR = { + "id": "PART_OF_FLOOR", + "keys": [ + {"label": "Room", "property": "floor_id"}, + {"label": "Floor", "property": "id"}, + ], +} +PART_OF_BUILDING = { + "id": "PART_OF_BUILDING", + "keys": [ + {"label": "Floor", "property": "building_id"}, + {"label": "Building", "property": "id"}, + ], +} + +await drasi.add_query( + "building-comfort-ui", CYPHER, ["db"], joins=[PART_OF_FLOOR, PART_OF_BUILDING] +) +``` + +With those joins declared, a query can match the whole hierarchy: + +```cypher +MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor)-[:PART_OF_BUILDING]->(b:Building) +``` + +#### Aggregating up the hierarchy + +Half of the queries don't just read rooms — they **roll comfort levels up** the building. In a Continuous Query, an aggregate such as `avg()` inside a `WITH` groups by whatever non-aggregated values travel alongside it, exactly like SQL's `GROUP BY`. + +**One stage — average a floor's rooms.** `floor-comfort-level-calc` keeps the floor `f` in the `WITH`, so `avg()` produces one value per floor: + +```cypher +WITH f, floor( 50 + (r.temperature - 72) + ... ) AS RoomComfortLevel +WITH f, avg(RoomComfortLevel) AS ComfortLevel +RETURN f.id AS FloorId, ComfortLevel +``` + +**Two stages — average the averages.** `building-alert` aggregates twice. Carrying `f, b` groups the first `avg()` by floor; dropping `f` from the next `WITH` widens the grouping so the second `avg()` rolls the floor averages up into a single building level: + +```cypher +WITH f, b, avg(RoomComfortLevel) AS FloorComfortLevel // rooms -> floor +WITH b, avg(FloorComfortLevel) AS ComfortLevel // floors -> building +``` + +**Filtering on an aggregate.** The `floor-alert` and `building-alert` queries place a `WHERE` *after* the aggregation — like SQL's `HAVING` — so a floor or the building only shows up while its average is outside the comfortable band: + +```cypher +WITH f, avg(RoomComfortLevel) AS ComfortLevel +WHERE ComfortLevel < 40 OR ComfortLevel > 50 +RETURN f.id AS FloorId, ComfortLevel +``` + +Because these are *continuous* queries, the aggregates are maintained **incrementally**: when one room's reading changes, Drasi recomputes only the affected floor and building averages and emits just that change — it never rescans every room. + +### The Python Reaction + +This is where the Python demo differs from a built-in dashboard. Instead of configuring a dashboard reaction, we register a **Python reaction** — a plain callback subscribed to all six queries. Each time a query's result set changes, Drasi calls it with the query id and the diffs, and the callback updates a thread-safe snapshot: + +```python +def on_results(event): + query = QUERIES_BY_ID[event["query_id"]] + with lock: + store = snapshot[query.id] # this query's rows, keyed by id; the UI reads it + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[query.key], None) + case {"after": dict() as row} | { + "data": dict() as row + }: # ADD / UPDATE / aggregation + store[row[query.key]] = row + + +await drasi.add_python_reaction("ui", [q.id for q in QUERIES], on_results) +``` + +Each query knows the RETURN column that identifies its rows (`query.key` — `RoomId`, `FloorId` or `BuildingId`). Every diff carries the affected row in `data` (for an `UPDATE` it equals `after`), so the snapshot is kept by row identity — upsert on `ADD`/`UPDATE`, drop on `DELETE`. The `type` is one of `ADD`, `UPDATE`, or `DELETE`. This works the same for the per-room queries and the aggregate floor/building queries, which emit an `UPDATE` as their average changes. See [Python reactions](../../guides/python-reactions/) for the full callback contract. + +### Embedding the engine in Streamlit + +Streamlit re-runs its script top to bottom on every interaction, which doesn't suit a long-lived async engine. So the engine runs on its own asyncio event loop in a **background daemon thread**, created exactly once with `@st.cache_resource` and shared across every browser session: + +```python +@st.cache_resource +def get_engine(): + engine = ComfortEngine() # spins up the background loop + Drasi + engine.wait_ready() + return engine +``` + +Each run, `app.py` reads `engine.snapshot()` and renders the building, the meter, the floor table and the alert lists. A one-and-a-half-second auto-refresh keeps the view live. Because the reaction only updates the snapshot when a query actually changes, the UI reflects each `UPDATE` the moment it lands — no polling of the database. + +The centerpiece building panel is just a loop over the `building-comfort-ui` rows, grouped by floor, picking a status emoji from each room's comfort level: + +```python +for floor_name in sorted(by_floor): + st.markdown(f"### {floor_name}") + for room in by_floor[floor_name]: + emoji, label = comfort_status(room["ComfortLevel"]) + st.markdown( + f"- **{room['RoomName']}** — comfort **{room['ComfortLevel']}** {emoji} {label}" + ) +``` + +And the reaction renders it live: + +<img src="images/streamlit-building-view.png" width="820" alt="The Building Comfort UI: rooms grouped by floor, each tagged with a comfort level and a green, red, or blue status emoji, with a sidebar of controls"> + +### Other ways to consume a query + +A Python reaction is the most direct way to turn query changes into a UI, but it isn't the only option `drasi-lib` gives you: + +- **Stream the changes** with `async for event in await drasi.query_results("room-alert")` and drive any front end you like. See [Streaming results](../../guides/streaming/). +- **Reuse a published reaction** such as the dashboard reaction plugin, exactly as the [Postgres dashboard example](../../examples/postgres-dashboard/) does — no UI code at all. + +Because Drasi emits only what *changed* — never the full result set — whichever you choose stays live without polling. + +## Putting It All Together {#all} + +The sections above walked through the Drasi side one piece at a time. Here it is assembled into the single `engine.py` — the source, the two synthetic joins, the six queries, and the `ComfortEngine` that runs them and writes back. `app.py` (the Streamlit UI) imports `ComfortEngine`, the query ids, and the comfort band from it: + +```python +import asyncio +import os +import random +import threading +from dataclasses import dataclass, field + +os.environ.setdefault("RUST_LOG", "warn") # quiet the engine's INFO logging + +import psycopg +from drasi import Drasi + +# --- The source: the tutorial's PostgreSQL database --- +CONNECTION = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5732")), + "database": os.environ.get("POSTGRES_DATABASE", "building_comfort"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), + "sslMode": "prefer", +} +SOURCE_CONFIG = { + **CONNECTION, + "tables": ["public.Building", "public.Floor", "public.Room"], + "tableKeys": [ + {"table": "Building", "keyColumns": ["id"]}, + {"table": "Floor", "keyColumns": ["id"]}, + {"table": "Room", "keyColumns": ["id"]}, + ], + "slotName": "drasi_building_comfort_slot", + "publicationName": "drasi_building_comfort_pub", +} +BOOTSTRAP_CONFIG = {"kind": "postgres", **CONNECTION} +# The UI writes room updates through a psycopg connection built from CONNECTION +# (psycopg spells the database key "dbname"), so Drasi observes them via CDC. + +COMFORT_DEFAULTS = {"temperature": 70, "humidity": 40, "co2": 10} +COMFORT_MIN, COMFORT_MAX = 40, 50 + +# --- Two synthetic joins walk Room -> Floor -> Building --- +PART_OF_FLOOR = { + "id": "PART_OF_FLOOR", + "keys": [ + {"label": "Room", "property": "floor_id"}, + {"label": "Floor", "property": "id"}, + ], +} +PART_OF_BUILDING = { + "id": "PART_OF_BUILDING", + "keys": [ + {"label": "Floor", "property": "building_id"}, + {"label": "Building", "property": "id"}, + ], +} + + +@dataclass(frozen=True) +class Query: + id: str + key: str # the RETURN column that identifies each row + cypher: str + joins: list[dict] = field(default_factory=list) + + +# Every query computes the same comfort level; a value in 40-50 is comfortable. +QUERIES = [ + # building-comfort-ui: one row per room with its comfort level. Drives the + # building view; walks Room -> Floor -> Building through the two joins. + Query( + id="building-comfort-ui", + key="RoomId", + joins=[PART_OF_FLOOR, PART_OF_BUILDING], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor)-[:PART_OF_BUILDING]->(b:Building) + RETURN r.id AS RoomId, r.name AS RoomName, f.id AS FloorId, f.name AS FloorName, + b.name AS BuildingName, r.temperature AS Temperature, r.humidity AS Humidity, + r.co2 AS CO2, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS ComfortLevel + """, + ), + # floor-comfort-level-calc: each floor's comfort, the average of its rooms. + Query( + id="floor-comfort-level-calc", + key="FloorId", + joins=[PART_OF_FLOOR], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor) + WITH f, floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS RoomComfort + WITH f, avg(RoomComfort) AS ComfortLevel + RETURN f.id AS FloorId, ComfortLevel + """, + ), + # room-alert: only the rooms whose comfort is outside the 40-50 band. + Query( + id="room-alert", + key="RoomId", + cypher=""" + MATCH (r:Room) + WITH r.id AS RoomId, r.name AS RoomName, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS ComfortLevel + WHERE ComfortLevel < 40 OR ComfortLevel > 50 + RETURN RoomId, RoomName, ComfortLevel + """, + ), + # building-comfort-level-calc (average of floor averages), floor-alert, and + # building-alert follow the same formula with avg()/WHERE variations. +] + + +class ComfortEngine: + """Runs Drasi on a background loop and exposes a thread-safe snapshot.""" + + def __init__(self): + self._lock = threading.Lock() + self._results = {} # query id -> {row key -> row} + self._key_by_id = {q.id: q.key for q in QUERIES} + self._simulation = False + self._ready = threading.Event() + self._drasi = None + threading.Thread(target=self._run, daemon=True).start() + + def _run(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._start()) + self._ready.set() + loop.run_forever() + + async def _start(self): + self._drasi = await Drasi.create("building-comfort") + for plugin in ("source/postgres", "bootstrap/postgres"): + await self._drasi.install_plugin(plugin) + await self._drasi.start() + + await self._drasi.add_source("postgres", "db", SOURCE_CONFIG, bootstrap=BOOTSTRAP_CONFIG) + for q in QUERIES: + await self._drasi.add_query(q.id, q.cypher, ["db"], joins=q.joins or None) + for q in QUERIES: + await self._drasi.wait_for_query(q.id) + rows = await self._drasi.get_query_results(q.id) + self._results[q.id] = {r[q.key]: r for r in rows} + + await self._drasi.add_python_reaction("ui", [q.id for q in QUERIES], self._on_results) + + def _on_results(self, event): + key = self._key_by_id[event["query_id"]] + with self._lock: + store = self._results.setdefault(event["query_id"], {}) + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[key], None) + case {"after": dict() as row} | {"data": dict() as row}: + store[row[key]] = row + + def snapshot(self): + with self._lock: + return { + "simulation": self._simulation, + "results": {q: list(s.values()) for q, s in self._results.items()}, + } + + # set_room(id, t, h, co2) and reset_room(id=None) write UPDATEs to Postgres; + # set_simulation(on) toggles a background task that assigns a random room new + # readings every few seconds. Drasi observes each write via CDC. +``` + +That's the whole Drasi side in one file: one source, two synthetic joins up the Room -> Floor -> Building hierarchy, six continuous queries (four of them aggregations), and a reaction that keeps a snapshot the UI renders and drives. + +## Clean Up {#cleanup} + +When you're finished, stop the app with **Ctrl+C** in the terminal, then remove the database container: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +# Stop containers, keep data +bash scripts/cleanup.sh + +# Stop containers and delete the data volume +bash scripts/cleanup.sh --volumes +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +# Stop containers, keep data +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 + +# Stop containers and delete the data volume +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 -RemoveVolumes +{{< /tab >}} +{{< /tabpane >}} + +## What You Learned {#summary} + +- **Sources** connect Drasi to live data — here, PostgreSQL via Change Data Capture, configured in a few lines of Python. +- **Continuous Queries** with **synthetic joins** let you model relationships (room → floor → building) and compute derived values (comfort levels) that stay current automatically. +- **Reactions** turn query changes into action. A **Python reaction** gave you a live UI from a callback and a snapshot — no bespoke web server, and complete control over the rendering. +- Because Drasi emits only what *changed*, everything updates the instant the data does, with no polling. + +From here, try editing the comfort formula or the layout in `app.py`, adding a new alert query, or pointing the source at your own database. diff --git a/tutorials/building-comfort/app.py b/tutorials/building-comfort/app.py new file mode 100644 index 0000000..13f0c2c --- /dev/null +++ b/tutorials/building-comfort/app.py @@ -0,0 +1,227 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Building Comfort -- a Streamlit UI driven by a Drasi Python reaction. + +Run it with: + + streamlit run app.py + +The engine (Drasi + the six continuous queries + one Python reaction) is created +once and shared across every browser session by ``@st.cache_resource``. This +script only *reads* the reaction's snapshot and renders it, and offers controls +that write room changes back to Postgres for Drasi to observe via CDC. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Any + +import streamlit as st +from engine import ( + BUILDING_COMFORT_UI, + COMFORT_MAX, + COMFORT_MIN, + FLOOR_ALERT, + FLOOR_COMFORT_LEVEL, + ROOM_ALERT, + ComfortEngine, +) +from streamlit_autorefresh import st_autorefresh + +st.set_page_config(page_title="Building Comfort", page_icon="🏢", layout="wide") + + +@st.cache_resource +def get_engine() -> ComfortEngine: + """Create (once) and return the shared Drasi engine. + + The spinner shows only on the very first run, while the engine downloads its + plugins, connects to Postgres and bootstraps the queries. + """ + engine = ComfortEngine() + with st.spinner("Starting Drasi: installing plugins and connecting to Postgres..."): + engine.wait_ready() + return engine + + +def comfort_status(level: float) -> tuple[str, str]: + """Map a comfort level to a status emoji and label.""" + if level > COMFORT_MAX: + return "🔴", "too hot" + if level < COMFORT_MIN: + return "🔵", "too cold" + return "🟢", "comfortable" + + +def render_building(rows: list[dict[str, Any]]) -> None: + st.subheader("Building Comfort") + if not rows: + st.info("Waiting for room data...") + return + + by_floor: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + by_floor[row["FloorName"]].append(row) + + for floor_name in sorted(by_floor): + st.markdown(f"### {floor_name}") + for room in sorted(by_floor[floor_name], key=lambda r: r["RoomName"]): + level = room["ComfortLevel"] + emoji, label = comfort_status(level) + st.markdown( + f"- **{room['RoomName']}** — comfort **{level:g}** {emoji} {label}" + f" · 🌡️ {room['Temperature']}°F" + f" 💧 {room['Humidity']}%" + f" 🫧 {room['CO2']} ppm" + ) + + +def render_overview(rows: list[dict[str, Any]]) -> None: + st.subheader("Overall") + if not rows: + st.metric("Building comfort", "—") + return + + average = sum(r["ComfortLevel"] for r in rows) / len(rows) + emoji, label = comfort_status(average) + st.metric("Building comfort", f"{average:.0f}", help="Average of every room") + st.caption(f"{emoji} {label} · comfortable band is {COMFORT_MIN}–{COMFORT_MAX}") + # A simple gauge: the comfort level on a 0–100 scale. + st.progress(min(max(average / 100.0, 0.0), 1.0)) + + +def render_floor_comfort(floor_rows: list[dict[str, Any]], ui_rows: list[dict[str, Any]]) -> None: + st.subheader("Floor comfort") + if not floor_rows: + st.info("Waiting for floor data...") + return + + names = {r["FloorId"]: r["FloorName"] for r in ui_rows} + table = [ + { + "Floor": names.get(r["FloorId"], r["FloorId"]), + "Comfort level": round(r["ComfortLevel"], 1), + } + for r in sorted(floor_rows, key=lambda r: r["FloorId"]) + ] + st.dataframe(table, hide_index=True, use_container_width=True) + + +def render_alerts(room_alerts: list[dict[str, Any]], floor_alerts: list[dict[str, Any]]) -> None: + st.subheader("Comfort alerts") + if room_alerts: + for a in sorted(room_alerts, key=lambda r: r["RoomId"]): + st.markdown( + f"- ⚠️ **{a['RoomName']}** (`{a['RoomId']}`) — comfort **{a['ComfortLevel']:g}**" + ) + else: + st.success("All rooms are comfortable.") + + st.subheader("Floor alerts") + if floor_alerts: + for a in sorted(floor_alerts, key=lambda r: r["FloorId"]): + st.markdown( + f"- ⚠️ **{a['FloorName']}** (`{a['FloorId']}`) — comfort **{a['ComfortLevel']:g}**" + ) + else: + st.success("All floors are comfortable.") + + +def render_controls(engine: ComfortEngine, snapshot: dict[str, Any]) -> None: + st.sidebar.header("Drive the demo") + st.sidebar.caption( + "These controls write SQL UPDATEs to Postgres. Drasi observes them via " + "CDC and the view updates on its own — there is no middle tier." + ) + + # Simulation mode. + simulate = st.sidebar.toggle("Simulation mode", value=snapshot["simulation"]) + if simulate != snapshot["simulation"]: + engine.set_simulation(simulate) + st.rerun() + if simulate: + st.sidebar.caption("Assigning a random room new readings every few seconds.") + + st.sidebar.divider() + + # Reset everything. + if st.sidebar.button("Reset all rooms", use_container_width=True): + engine.reset_room() + st.toast("Reset every room to comfortable.") + + st.sidebar.divider() + + # Per-room controls. + ui_rows = snapshot["results"].get(BUILDING_COMFORT_UI, []) + if not ui_rows: + return + + by_id = {r["RoomId"]: r for r in ui_rows} + options = sorted(by_id, key=lambda rid: (by_id[rid]["FloorName"], by_id[rid]["RoomName"])) + + st.sidebar.subheader("Set a room") + room_id = st.sidebar.selectbox( + "Room", + options, + format_func=lambda rid: f"{by_id[rid]['FloorName']} / {by_id[rid]['RoomName']}", + ) + current = by_id[room_id] + + with st.sidebar.form("set-room"): + temperature = st.number_input("Temperature (°F)", value=int(current["Temperature"])) + humidity = st.number_input("Humidity (%)", value=int(current["Humidity"])) + co2 = st.number_input("CO₂ (ppm)", value=int(current["CO2"])) + submitted = st.form_submit_button("Set room", use_container_width=True) + if submitted: + engine.set_room(room_id, int(temperature), int(humidity), int(co2)) + st.toast(f"Updated {current['RoomName']}.") + + if st.sidebar.button("Reset this room", use_container_width=True): + engine.reset_room(room_id) + st.toast(f"Reset {current['RoomName']} to comfortable.") + + +def main() -> None: + engine = get_engine() + + # Re-run every 1.5s so the reaction's latest snapshot is rendered live. + st_autorefresh(interval=1500, key="comfort-refresh") + + snapshot = engine.snapshot() + if snapshot["error"]: + st.error(f"The Drasi engine failed: {snapshot['error']}") + return + + results = snapshot["results"] + ui_rows = results.get(BUILDING_COMFORT_UI, []) + building_name = ui_rows[0]["BuildingName"] if ui_rows else "Building Comfort" + + st.title(f"🏢 {building_name}") + + render_controls(engine, snapshot) + + left, right = st.columns([2, 1]) + with left: + render_building(ui_rows) + with right: + render_overview(ui_rows) + render_alerts(results.get(ROOM_ALERT, []), results.get(FLOOR_ALERT, [])) + + render_floor_comfort(results.get(FLOOR_COMFORT_LEVEL, []), ui_rows) + + +if __name__ == "__main__": + main() diff --git a/tutorials/building-comfort/database/docker-compose.yml b/tutorials/building-comfort/database/docker-compose.yml new file mode 100644 index 0000000..b87c9af --- /dev/null +++ b/tutorials/building-comfort/database/docker-compose.yml @@ -0,0 +1,44 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: building-comfort + +services: + postgres: + image: postgres:14-alpine + container_name: building-comfort-postgres + environment: + POSTGRES_DB: building_comfort + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres_admin + command: + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_replication_slots=10" + - "-c" + - "max_wal_senders=10" + ports: + - "${POSTGRES_HOST_PORT:-5732}:5432" + volumes: + - building_comfort_postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d building_comfort"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + building_comfort_postgres_data: diff --git a/tutorials/building-comfort/database/init.sql b/tutorials/building-comfort/database/init.sql new file mode 100644 index 0000000..41e3ed8 --- /dev/null +++ b/tutorials/building-comfort/database/init.sql @@ -0,0 +1,146 @@ +-- Copyright 2026 The Drasi Authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Building Comfort Tutorial Database Schema +-- +-- Models a smart building as Building -> Floor -> Room. Each Room carries +-- temperature, humidity and co2 sensor readings. Table names are quoted and +-- PascalCase ("Building", "Floor", "Room") so that the node labels Drasi sees +-- match the Cypher continuous queries (which use (r:Room), (f:Floor), +-- (b:Building)) without any change to the query text. + +-- Suppress noisy output during setup. +\set QUIET on +SET client_min_messages = ERROR; + +-- Create a user with replication privileges for CDC. +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'drasi_user') THEN + CREATE USER drasi_user WITH REPLICATION LOGIN PASSWORD 'drasi_password'; + END IF; +END +$$; + +-- Grant permissions on the database. +GRANT CREATE ON DATABASE building_comfort TO drasi_user; +GRANT ALL PRIVILEGES ON DATABASE building_comfort TO drasi_user; + +-- Drop existing tables if they exist. +DROP TABLE IF EXISTS "Room" CASCADE; +DROP TABLE IF EXISTS "Floor" CASCADE; +DROP TABLE IF EXISTS "Building" CASCADE; + +-- Building table. +CREATE TABLE "Building" ( + id VARCHAR(50) PRIMARY KEY, + name VARCHAR(100) NOT NULL +); + +-- Floor table (each Floor belongs to a Building). +CREATE TABLE "Floor" ( + id VARCHAR(50) PRIMARY KEY, + name VARCHAR(100) NOT NULL, + building_id VARCHAR(50) NOT NULL, + FOREIGN KEY (building_id) REFERENCES "Building"(id) ON DELETE CASCADE +); + +-- Room table (each Room belongs to a Floor and holds sensor readings). +CREATE TABLE "Room" ( + id VARCHAR(50) PRIMARY KEY, + name VARCHAR(100) NOT NULL, + temperature INTEGER, + humidity INTEGER, + co2 INTEGER, + floor_id VARCHAR(50) NOT NULL, + FOREIGN KEY (floor_id) REFERENCES "Floor"(id) ON DELETE CASCADE +); + +-- Set REPLICA IDENTITY to FULL so change events include every column. +ALTER TABLE "Building" REPLICA IDENTITY FULL; +ALTER TABLE "Floor" REPLICA IDENTITY FULL; +ALTER TABLE "Room" REPLICA IDENTITY FULL; + +-- Ensure drasi_user owns the tables. +ALTER TABLE "Building" OWNER TO drasi_user; +ALTER TABLE "Floor" OWNER TO drasi_user; +ALTER TABLE "Room" OWNER TO drasi_user; + +-- Grant permissions to drasi_user. +GRANT USAGE ON SCHEMA public TO drasi_user; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO drasi_user; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO drasi_user; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO drasi_user; + +-- Create the publication for logical replication. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'drasi_building_comfort_pub') THEN + CREATE PUBLICATION drasi_building_comfort_pub FOR TABLE "Building", "Floor", "Room"; + END IF; +END +$$; + +-- Seed the building hierarchy: 1 building, 3 floors, 9 rooms. +-- Every room starts comfortable (temperature=70, humidity=40, co2=10), +-- which yields a comfort level of floor(50 + (70-72) + (40-42) + 0) = 46. +-- Seeding happens BEFORE the replication slot is created so the existing rows +-- are loaded via bootstrap and not replayed as change events. +INSERT INTO "Building" (id, name) +SELECT * FROM (VALUES ('building_01', 'Building 01')) AS d(id, name) +WHERE NOT EXISTS (SELECT 1 FROM "Building"); + +INSERT INTO "Floor" (id, name, building_id) +SELECT * FROM (VALUES + ('floor_01_01', 'Floor 01', 'building_01'), + ('floor_01_02', 'Floor 02', 'building_01'), + ('floor_01_03', 'Floor 03', 'building_01') +) AS d(id, name, building_id) +WHERE NOT EXISTS (SELECT 1 FROM "Floor"); + +INSERT INTO "Room" (id, name, temperature, humidity, co2, floor_id) +SELECT * FROM (VALUES + ('room_01_01_01', 'Room 01', 70, 40, 10, 'floor_01_01'), + ('room_01_01_02', 'Room 02', 70, 40, 10, 'floor_01_01'), + ('room_01_01_03', 'Room 03', 70, 40, 10, 'floor_01_01'), + ('room_01_02_01', 'Room 01', 70, 40, 10, 'floor_01_02'), + ('room_01_02_02', 'Room 02', 70, 40, 10, 'floor_01_02'), + ('room_01_02_03', 'Room 03', 70, 40, 10, 'floor_01_02'), + ('room_01_03_01', 'Room 01', 70, 40, 10, 'floor_01_03'), + ('room_01_03_02', 'Room 02', 70, 40, 10, 'floor_01_03'), + ('room_01_03_03', 'Room 03', 70, 40, 10, 'floor_01_03') +) AS d(id, name, temperature, humidity, co2, floor_id) +WHERE NOT EXISTS (SELECT 1 FROM "Room"); + +-- Create the replication slot AFTER seeding. The slot captures only changes +-- made after this point; existing rows are retrieved via bootstrap when the +-- queries start, so the seed data is not double-counted. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'drasi_building_comfort_slot') THEN + PERFORM pg_create_logical_replication_slot('drasi_building_comfort_slot', 'pgoutput'); + END IF; +END +$$; + +-- Summary. +SET client_min_messages = NOTICE; +DO $$ +BEGIN + RAISE NOTICE 'Building Comfort database initialized successfully!'; + RAISE NOTICE 'Tables: Building, Floor, Room'; + RAISE NOTICE 'Publication: drasi_building_comfort_pub'; + RAISE NOTICE 'Replication slot: drasi_building_comfort_slot'; +END +$$; diff --git a/tutorials/building-comfort/engine.py b/tutorials/building-comfort/engine.py new file mode 100644 index 0000000..9914dac --- /dev/null +++ b/tutorials/building-comfort/engine.py @@ -0,0 +1,457 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The embedded Drasi engine that powers the Streamlit UI. + +This one file holds everything behind the demo: the configuration, the six +continuous queries, and the engine that runs them. + +Streamlit re-runs its script top to bottom on every interaction, which is a poor +fit for a long-lived async engine. So the Drasi engine runs on its own asyncio +event loop in a background daemon thread, created exactly once (Streamlit caches +it with ``@st.cache_resource``). The engine: + +1. installs the ``source/postgres`` and ``bootstrap/postgres`` plugins, +2. adds the Postgres source and the six continuous queries, and +3. registers a single **Python reaction** over all six queries. + +The reaction keeps a thread-safe snapshot of every query's result set. Streamlit +reads that snapshot each run and renders it; there is no dashboard or SSE +reaction, and no bespoke web server -- the reaction *is* the application. + +The UI's controls (set a room, reset, simulate) write plain SQL ``UPDATE``s to +Postgres via ``psycopg``. Drasi observes those writes through logical +replication and re-runs the affected queries, which calls the reaction, which +updates the snapshot -- the same path a real building-management app would take. +""" + +from __future__ import annotations + +import os + +# Quiet the engine's default INFO logging so it doesn't swamp Streamlit's +# console. Set RUST_LOG=info (or debug) to watch what the engine is doing. This +# must happen before ``drasi`` is imported. +os.environ.setdefault("RUST_LOG", "warn") + +import asyncio # noqa: E402 +import random # noqa: E402 +import threading # noqa: E402 +from dataclasses import dataclass, field # noqa: E402 +from typing import Any # noqa: E402 + +import psycopg # noqa: E402 + +from drasi import Drasi # noqa: E402 +from drasi.types import QueryResultEvent # noqa: E402 + +# ============================================================================= +# Configuration -- everything read from the environment with defaults. +# ============================================================================= + +POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost") +POSTGRES_PORT = int(os.environ.get("POSTGRES_PORT", "5732")) +POSTGRES_DATABASE = os.environ.get("POSTGRES_DATABASE", "building_comfort") +POSTGRES_USER = os.environ.get("POSTGRES_USER", "drasi_user") +POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "drasi_password") +POSTGRES_SSLMODE = os.environ.get("POSTGRES_SSLMODE", "prefer") + +# The Drasi source plugin spells the connection keys the same way the Node.js +# binding does (camelCase); Drasi passes plugin config through untouched. +SOURCE_CONNECTION = { + "host": POSTGRES_HOST, + "port": POSTGRES_PORT, + "database": POSTGRES_DATABASE, + "user": POSTGRES_USER, + "password": POSTGRES_PASSWORD, + "sslMode": POSTGRES_SSLMODE, +} + +# psycopg spells the database key ``dbname`` rather than ``database``. The demo +# writes room updates directly to Postgres through this connection so Drasi +# observes them via CDC -- exactly as a real building-management app would. +PSYCOPG_CONNECTION = { + "host": POSTGRES_HOST, + "port": POSTGRES_PORT, + "dbname": POSTGRES_DATABASE, + "user": POSTGRES_USER, + "password": POSTGRES_PASSWORD, +} + +# The Postgres source replicates these three tables. They are schema-qualified +# here (``public.Room``) while ``tableKeys`` uses the bare table name; the node +# label Drasi sees is the bare name, so the Cypher matches (r:Room) etc. +SOURCE_CONFIG = { + **SOURCE_CONNECTION, + "tables": ["public.Building", "public.Floor", "public.Room"], + "tableKeys": [ + {"table": "Building", "keyColumns": ["id"]}, + {"table": "Floor", "keyColumns": ["id"]}, + {"table": "Room", "keyColumns": ["id"]}, + ], + "slotName": "drasi_building_comfort_slot", + "publicationName": "drasi_building_comfort_pub", +} + +# The bootstrap provider loads the rows that already exist when the query starts, +# so the dashboard is populated before any change arrives. +BOOTSTRAP_CONFIG = {"kind": "postgres", **SOURCE_CONNECTION} + +# Comfortable defaults. floor(50 + (70-72) + (40-42) + 0) = 46, inside 40-50. +COMFORT_DEFAULTS = {"temperature": 70, "humidity": 40, "co2": 10} + +# The comfortable band. A room, floor or building outside it raises an alert. +COMFORT_MIN = 40 +COMFORT_MAX = 50 + +# How often simulation mode assigns a random room new readings, in seconds. +SIMULATION_INTERVAL_S = float(os.environ.get("SIMULATION_INTERVAL_S", "3")) + +# ============================================================================= +# The six continuous queries. +# ============================================================================= +# +# Each query is written out in full so you can read (or copy) exactly what Drasi +# runs. They all compute the same comfort level: +# +# floor(50 + (temperature - 72) + (humidity - 42) +# + CASE WHEN co2 > 500 THEN (co2 - 500) / 25 ELSE 0 END) +# +# A value between 40 and 50 is comfortable; the seed values (70F, 40%, 10 ppm) +# give floor(50 + (70-72) + (40-42) + 0) = 46. +# +# The Room -> Floor -> Building hierarchy is walked through two *synthetic +# joins*. Drasi does not read Postgres foreign keys, so each query declares the +# relationships it needs -- Room.floor_id -> Floor.id, and Floor.building_id -> +# Building.id -- as a plain mapping passed to ``add_query``. + +PART_OF_FLOOR = { + "id": "PART_OF_FLOOR", + "keys": [ + {"label": "Room", "property": "floor_id"}, + {"label": "Floor", "property": "id"}, + ], +} + +PART_OF_BUILDING = { + "id": "PART_OF_BUILDING", + "keys": [ + {"label": "Floor", "property": "building_id"}, + {"label": "Building", "property": "id"}, + ], +} + +# Query ids -- the UI reads each query's result set from the snapshot by these. +BUILDING_COMFORT_UI = "building-comfort-ui" +BUILDING_COMFORT_LEVEL = "building-comfort-level-calc" +FLOOR_COMFORT_LEVEL = "floor-comfort-level-calc" +ROOM_ALERT = "room-alert" +FLOOR_ALERT = "floor-alert" +BUILDING_ALERT = "building-alert" + + +@dataclass(frozen=True) +class Query: + """One continuous query: how to register it and how to index its rows.""" + + id: str + key: str # the RETURN column that identifies each row (its primary key) + cypher: str + joins: list[dict[str, Any]] = field(default_factory=list) + + +QUERIES = [ + # One row per room, with its comfort level. Drives the building view. + Query( + id=BUILDING_COMFORT_UI, + key="RoomId", + joins=[PART_OF_FLOOR, PART_OF_BUILDING], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor)-[:PART_OF_BUILDING]->(b:Building) + RETURN + r.id AS RoomId, + r.name AS RoomName, + f.id AS FloorId, + f.name AS FloorName, + b.id AS BuildingId, + b.name AS BuildingName, + r.temperature AS Temperature, + r.humidity AS Humidity, + r.co2 AS CO2, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS ComfortLevel + """, + ), + # The building's overall comfort: the average of each floor's average. + Query( + id=BUILDING_COMFORT_LEVEL, + key="BuildingId", + joins=[PART_OF_FLOOR, PART_OF_BUILDING], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor)-[:PART_OF_BUILDING]->(b:Building) + WITH b, f, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS RoomComfortLevel + WITH b, avg(RoomComfortLevel) AS FloorComfortLevel + WITH b, avg(FloorComfortLevel) AS ComfortLevel + RETURN b.id AS BuildingId, ComfortLevel + """, + ), + # Each floor's comfort: the average of the rooms on it. + Query( + id=FLOOR_COMFORT_LEVEL, + key="FloorId", + joins=[PART_OF_FLOOR], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor) + WITH f, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS RoomComfortLevel + WITH f, avg(RoomComfortLevel) AS ComfortLevel + RETURN f.id AS FloorId, ComfortLevel + """, + ), + # Only the rooms whose comfort is outside the 40-50 band. + Query( + id=ROOM_ALERT, + key="RoomId", + cypher=""" + MATCH (r:Room) + WITH r.id AS RoomId, r.name AS RoomName, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS ComfortLevel + WHERE ComfortLevel < 40 OR ComfortLevel > 50 + RETURN RoomId, RoomName, ComfortLevel + """, + ), + # Only the floors whose average comfort is outside the 40-50 band. + Query( + id=FLOOR_ALERT, + key="FloorId", + joins=[PART_OF_FLOOR], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor) + WITH f, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS RoomComfortLevel + WITH f, avg(RoomComfortLevel) AS ComfortLevel + WHERE ComfortLevel < 40 OR ComfortLevel > 50 + RETURN f.id AS FloorId, f.name AS FloorName, ComfortLevel + """, + ), + # The building, only while its overall comfort is outside the 40-50 band. + Query( + id=BUILDING_ALERT, + key="BuildingId", + joins=[PART_OF_FLOOR, PART_OF_BUILDING], + cypher=""" + MATCH (r:Room)-[:PART_OF_FLOOR]->(f:Floor)-[:PART_OF_BUILDING]->(b:Building) + WITH b, f, + floor(50 + (r.temperature - 72) + (r.humidity - 42) + + CASE WHEN r.co2 > 500 THEN (r.co2 - 500) / 25 ELSE 0 END) AS RoomComfortLevel + WITH b, f, avg(RoomComfortLevel) AS FloorComfortLevel + WITH b, avg(FloorComfortLevel) AS ComfortLevel + WHERE ComfortLevel < 40 OR ComfortLevel > 50 + RETURN b.id AS BuildingId, b.name AS BuildingName, ComfortLevel + """, + ), +] + +# ============================================================================= +# The engine. +# ============================================================================= + +# Ranges chosen to straddle the comfortable band, so simulation makes alerts +# come and go rather than sitting at one extreme. +_SIM_TEMPERATURE = (55, 85) +_SIM_HUMIDITY = (20, 55) +_SIM_CO2 = (5, 904) + + +def _update_rooms(sql: str, params: tuple[Any, ...] = ()) -> None: + """Run one UPDATE against the Room table on a short-lived connection.""" + with psycopg.connect(**PSYCOPG_CONNECTION) as conn: + with conn.cursor() as cur: + cur.execute(sql, params) + + +class ComfortEngine: + """Runs Drasi on a background loop and exposes a thread-safe snapshot.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + # Per query: a dict keyed by that query's primary-key field. The reaction + # applies diffs into it; the UI reads a snapshot of the values. + self._results: dict[str, dict[Any, dict[str, Any]]] = {} + # Which RETURN column identifies each query's rows, looked up by query id. + self._key_by_id = {query.id: query.key for query in QUERIES} + self._version = 0 + self._simulation = False + + self._loop: asyncio.AbstractEventLoop | None = None + self._drasi: Drasi | None = None + self._sim_task: asyncio.Task[None] | None = None + self._room_ids: list[str] = [] + + self._ready = threading.Event() + self._error: BaseException | None = None + + self._thread = threading.Thread(target=self._run, name="drasi-comfort-engine", daemon=True) + self._thread.start() + + # -- lifecycle (background thread) --------------------------------------- + + def _run(self) -> None: + loop = asyncio.new_event_loop() + self._loop = loop + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(self._start()) + except BaseException as exc: # noqa: BLE001 - surfaced to the UI thread + self._error = exc + self._ready.set() + return + self._ready.set() + loop.run_forever() + + async def _start(self) -> None: + self._drasi = await Drasi.create("building-comfort") + + for plugin in ("source/postgres", "bootstrap/postgres"): + await self._drasi.install_plugin(plugin) + await self._drasi.start() + + await self._drasi.add_source("postgres", "db", SOURCE_CONFIG, bootstrap=BOOTSTRAP_CONFIG) + + for query in QUERIES: + await self._drasi.add_query(query.id, query.cypher, ["db"], joins=query.joins or None) + for query in QUERIES: + await self._drasi.wait_for_query(query.id) + + # Prime the snapshot from the bootstrapped result sets, then let the + # reaction keep it current. + for query in QUERIES: + rows = await self._drasi.get_query_results(query.id) + with self._lock: + self._results[query.id] = {row[query.key]: row for row in rows} + self._room_ids = await self._load_room_ids() + + await self._drasi.add_python_reaction( + "ui", [query.id for query in QUERIES], self._on_results + ) + + def _on_results(self, event: QueryResultEvent) -> None: + """Apply a batch of diffs to the snapshot. + + ``add_python_reaction`` calls this synchronously. Every diff carries the + affected row in ``data`` (for an UPDATE it equals ``after``), so the + snapshot is maintained by row identity: upsert on ADD/UPDATE, drop on + DELETE. Result sets are tiny, so this stays cheap. + """ + query_id = event["query_id"] + key_field = self._key_by_id[query_id] + with self._lock: + store = self._results.setdefault(query_id, {}) + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[key_field], None) + case {"after": dict() as row} | {"data": dict() as row}: + # ADD / UPDATE / aggregation: `after` is the new state + # (aggregation diffs carry a null `data`), `data` covers ADD. + store[row[key_field]] = row + self._version += 1 + + async def _load_room_ids(self) -> list[str]: + def _query() -> list[str]: + with psycopg.connect(**PSYCOPG_CONNECTION) as conn: + with conn.cursor() as cur: + cur.execute('SELECT id FROM "Room" ORDER BY id') + return [row[0] for row in cur.fetchall()] + + return await asyncio.get_running_loop().run_in_executor(None, _query) + + # -- readiness ----------------------------------------------------------- + + def wait_ready(self, timeout: float = 180.0) -> None: + """Block until the engine is running, raising if it failed to start.""" + if not self._ready.wait(timeout): + raise TimeoutError("the Drasi engine did not start in time") + if self._error is not None: + raise RuntimeError(f"the Drasi engine failed to start: {self._error}") + + # -- reads (any thread) -------------------------------------------------- + + def snapshot(self) -> dict[str, Any]: + """Return the current result set of every query, plus engine flags.""" + with self._lock: + results = {qid: list(store.values()) for qid, store in self._results.items()} + return { + "version": self._version, + "simulation": self._simulation, + "results": results, + "error": str(self._error) if self._error else None, + } + + # -- writes (any thread -> Postgres, observed via CDC) ------------------- + + def set_room(self, room_id: str, temperature: int, humidity: int, co2: int) -> None: + _update_rooms( + 'UPDATE "Room" SET temperature = %s, humidity = %s, co2 = %s WHERE id = %s', + (temperature, humidity, co2, room_id), + ) + + def reset_room(self, room_id: str | None = None) -> None: + """Reset one room, or every room when ``room_id`` is None.""" + d = COMFORT_DEFAULTS + if room_id is None: + _update_rooms( + 'UPDATE "Room" SET temperature = %s, humidity = %s, co2 = %s', + (d["temperature"], d["humidity"], d["co2"]), + ) + else: + self.set_room(room_id, d["temperature"], d["humidity"], d["co2"]) + + def set_simulation(self, enabled: bool) -> None: + """Turn hands-free simulation on or off.""" + if self._loop is None: + return + future = asyncio.run_coroutine_threadsafe(self._set_simulation(enabled), self._loop) + future.result(timeout=10) + + async def _set_simulation(self, enabled: bool) -> None: + if enabled and self._sim_task is None: + if not self._room_ids: + self._room_ids = await self._load_room_ids() + self._sim_task = asyncio.create_task(self._simulate()) + self._simulation = True + elif not enabled and self._sim_task is not None: + self._sim_task.cancel() + self._sim_task = None + self._simulation = False + + async def _simulate(self) -> None: + loop = asyncio.get_running_loop() + try: + while True: + room = random.choice(self._room_ids) + temperature = random.randint(*_SIM_TEMPERATURE) + humidity = random.randint(*_SIM_HUMIDITY) + co2 = random.randint(*_SIM_CO2) + # Run the blocking write off the event loop so the reaction keeps + # delivering while simulation is on. + await loop.run_in_executor(None, self.set_room, room, temperature, humidity, co2) + await asyncio.sleep(SIMULATION_INTERVAL_S) + except asyncio.CancelledError: + pass diff --git a/tutorials/building-comfort/images/streamlit-building-view.png b/tutorials/building-comfort/images/streamlit-building-view.png new file mode 100644 index 0000000..9c55229 Binary files /dev/null and b/tutorials/building-comfort/images/streamlit-building-view.png differ diff --git a/tutorials/building-comfort/requirements.txt b/tutorials/building-comfort/requirements.txt new file mode 100644 index 0000000..9c5b83b --- /dev/null +++ b/tutorials/building-comfort/requirements.txt @@ -0,0 +1,11 @@ +# Building Comfort tutorial dependencies. +# +# pip install -r requirements.txt +# +# drasi-lib is the Python binding for Drasi's embeddable engine. It downloads the +# Postgres source/bootstrap plugins from ghcr.io at runtime, so the first run +# needs network access. +drasi-lib>=0.2,<1 +streamlit>=1.32 +streamlit-autorefresh>=1.0 +psycopg[binary]>=3.1 diff --git a/tutorials/building-comfort/scripts/cleanup.ps1 b/tutorials/building-comfort/scripts/cleanup.ps1 new file mode 100644 index 0000000..a3f7382 --- /dev/null +++ b/tutorials/building-comfort/scripts/cleanup.ps1 @@ -0,0 +1,48 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup Script (Windows) +# Stops the PostgreSQL container. The app runs in the foreground, so stop it +# first with Ctrl+C. Pass -RemoveVolumes to also delete the database volume. + +param( + [switch]$RemoveVolumes +) + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$DatabaseDir = Join-Path $ScriptDir "..\database" + +Write-Host "=== Building Comfort - Cleanup ===" +Write-Host "" + +Push-Location $DatabaseDir +try { + if ($RemoveVolumes) { + Write-Host "Removing container and volumes..." + docker compose down -v 2>&1 | Write-Host + } else { + Write-Host "Removing container (keeping volumes)..." + docker compose down 2>&1 | Write-Host + } +} +finally { + Pop-Location +} + +Write-Host "" +Write-Host "=== Cleanup complete! ===" +Write-Host "" +Write-Host "Options:" +Write-Host " scripts/cleanup.ps1 # Stop containers, keep data volumes" +Write-Host " scripts/cleanup.ps1 -RemoveVolumes # Stop containers and remove data volumes" diff --git a/tutorials/building-comfort/scripts/cleanup.sh b/tutorials/building-comfort/scripts/cleanup.sh new file mode 100755 index 0000000..852b4a0 --- /dev/null +++ b/tutorials/building-comfort/scripts/cleanup.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup Script +# Removes the PostgreSQL container. The app runs in the foreground, so stop it +# first with Ctrl+C. Pass --volumes to also delete the database volume. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATABASE_DIR="$SCRIPT_DIR/../database" +REMOVE_VOLUMES="${1:-}" + +echo "=== Building Comfort - Cleanup ===" +echo + +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null 2>&1; then + COMPOSE_CMD="docker compose" +else + echo "Warning: docker-compose not found, skipping container cleanup" + exit 0 +fi + +echo "Stopping PostgreSQL container..." +cd "$DATABASE_DIR" + +if [ "$REMOVE_VOLUMES" = "--volumes" ] || [ "$REMOVE_VOLUMES" = "-v" ]; then + echo "Removing container and volumes..." + $COMPOSE_CMD down -v +else + echo "Removing container (keeping volumes)..." + $COMPOSE_CMD down +fi + +echo +echo "=== Cleanup complete! ===" +echo +echo "Options:" +echo " $0 # Stop containers, keep data volumes" +echo " $0 --volumes # Stop containers and remove data volumes" diff --git a/tutorials/building-comfort/scripts/setup-database.ps1 b/tutorials/building-comfort/scripts/setup-database.ps1 new file mode 100644 index 0000000..98f02f0 --- /dev/null +++ b/tutorials/building-comfort/scripts/setup-database.ps1 @@ -0,0 +1,105 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup Database Script (Windows) +# Starts PostgreSQL with logical replication (WAL) enabled for CDC and seeds the +# building hierarchy (1 Building, 3 Floors, 9 Rooms). + +# Note: we deliberately do not set `$ErrorActionPreference = "Stop"` because the +# docker CLI writes progress to stderr, which PowerShell would otherwise treat +# as a terminating error. We check $LASTEXITCODE explicitly instead. + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$DatabaseDir = Join-Path $ScriptDir "..\database" + +Write-Host "=== Building Comfort - Database Setup ===" +Write-Host "" + +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + Write-Host "Error: Docker is not installed or not in PATH" + Write-Host "Please install Docker: https://docs.docker.com/get-docker/" + exit 1 +} + +docker info 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Docker daemon is not running" + Write-Host "Please start Docker and try again" + exit 1 +} + +Push-Location $DatabaseDir +try { + Write-Host "Stopping any existing PostgreSQL container..." + docker compose down -v 2>&1 | Out-Null + + Write-Host "Starting PostgreSQL with WAL replication..." + docker compose up -d 2>&1 | Write-Host + if ($LASTEXITCODE -ne 0) { + Write-Host "Error: failed to start PostgreSQL container" + exit 1 + } + + Write-Host "Waiting for PostgreSQL to be ready..." + $maxRetries = 30 + $ready = $false + for ($i = 1; $i -le $maxRetries; $i++) { + docker exec building-comfort-postgres pg_isready -U postgres -d building_comfort 2>&1 | Out-Null + if ($LASTEXITCODE -eq 0) { + Write-Host "PostgreSQL is ready!" + $ready = $true + break + } + Write-Host " Waiting... ($i/$maxRetries)" + Start-Sleep -Seconds 2 + } + + if (-not $ready) { + Write-Host "Error: PostgreSQL failed to start within the timeout" + Write-Host "Check logs with: docker logs building-comfort-postgres" + exit 1 + } + + # Apply the schema and seed data over stdin. We pipe init.sql into psql with + # `docker exec -i` instead of bind-mounting it into the container, so the same + # flow works for the dev container, Codespaces, and bare-metal runs. + Write-Host "Applying schema and seed data..." + Get-Content (Join-Path $DatabaseDir "init.sql") | ` + docker exec -i building-comfort-postgres psql -v ON_ERROR_STOP=1 -U postgres -d building_comfort + if ($LASTEXITCODE -ne 0) { + Write-Host "Error: failed to apply schema and seed data" + exit 1 + } + + Write-Host "" + Write-Host "Seeded rooms:" + # Pipe the query over stdin to avoid PowerShell mangling the embedded quotes + # around the case-sensitive "Room" identifier when passing -c to docker. + 'SELECT id, name, temperature, humidity, co2, floor_id FROM "Room" ORDER BY id;' | ` + docker exec -i building-comfort-postgres psql -U drasi_user -d building_comfort + + Write-Host "" + Write-Host "=== Database setup complete! ===" + Write-Host "" + $hostPort = if ($env:POSTGRES_HOST_PORT) { $env:POSTGRES_HOST_PORT } else { "5732" } + Write-Host "Connection details:" + Write-Host " Host: localhost" + Write-Host " Port: $hostPort" + Write-Host " Database: building_comfort" + Write-Host " User: drasi_user" + Write-Host " Password: drasi_password" +} +finally { + Pop-Location +} diff --git a/tutorials/building-comfort/scripts/setup-database.sh b/tutorials/building-comfort/scripts/setup-database.sh new file mode 100755 index 0000000..5f14b3a --- /dev/null +++ b/tutorials/building-comfort/scripts/setup-database.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup Database Script +# Starts PostgreSQL with logical replication (WAL) enabled for CDC and seeds the +# building hierarchy (1 Building, 3 Floors, 9 Rooms). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATABASE_DIR="$SCRIPT_DIR/../database" + +echo "=== Building Comfort - Database Setup ===" +echo + +if ! command -v docker &> /dev/null; then + echo "Error: Docker is not installed or not in PATH" + echo "Please install Docker: https://docs.docker.com/get-docker/" + exit 1 +fi + +if ! docker info &> /dev/null; then + echo "Error: Docker daemon is not running" + echo "Please start Docker and try again" + exit 1 +fi + +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null 2>&1; then + COMPOSE_CMD="docker compose" +else + echo "Error: docker-compose is not installed" + echo "Please install Docker Compose: https://docs.docker.com/compose/install/" + exit 1 +fi + +echo "Using: $COMPOSE_CMD" +echo + +echo "Stopping any existing PostgreSQL container..." +cd "$DATABASE_DIR" +$COMPOSE_CMD down -v 2>/dev/null || true + +echo "Starting PostgreSQL with WAL replication..." +$COMPOSE_CMD up -d + +echo "Waiting for PostgreSQL to be ready..." +MAX_RETRIES=30 +RETRY_COUNT=0 +POSTGRES_READY=false +while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + if docker exec building-comfort-postgres pg_isready -U postgres -d building_comfort &> /dev/null; then + echo "PostgreSQL is ready!" + POSTGRES_READY=true + break + fi + RETRY_COUNT=$((RETRY_COUNT + 1)) + echo " Waiting... ($RETRY_COUNT/$MAX_RETRIES)" + sleep 2 +done + +if [ "$POSTGRES_READY" != "true" ]; then + echo "Error: PostgreSQL did not become ready within the timeout." + echo "Check logs with: docker logs building-comfort-postgres" + exit 1 +fi + +if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then + echo "Error: PostgreSQL failed to start within the timeout" + echo "Check logs with: docker logs building-comfort-postgres" + exit 1 +fi + +# Apply the schema and seed data over stdin. We pipe init.sql into psql with +# `docker exec -i` instead of bind-mounting it into the container. A bind mount +# would break under docker-outside-of-docker (the dev container's file path does +# not exist on the host's Docker daemon); piping over stdin works the same way +# for the dev container, Codespaces, and bare-metal runs. +echo "Applying schema and seed data..." +docker exec -i building-comfort-postgres \ + psql -v ON_ERROR_STOP=1 -U postgres -d building_comfort < "$DATABASE_DIR/init.sql" + +echo +echo "Verifying database setup..." + +PUB_EXISTS=$(docker exec building-comfort-postgres psql -U drasi_user -d building_comfort -tAc \ + "SELECT 1 FROM pg_publication WHERE pubname = 'drasi_building_comfort_pub';" 2>/dev/null || echo "0") +if [ "$PUB_EXISTS" = "1" ]; then + echo " Publication: drasi_building_comfort_pub [OK]" +else + echo " Publication: drasi_building_comfort_pub [MISSING]" +fi + +SLOT_EXISTS=$(docker exec building-comfort-postgres psql -U drasi_user -d building_comfort -tAc \ + "SELECT 1 FROM pg_replication_slots WHERE slot_name = 'drasi_building_comfort_slot';" 2>/dev/null || echo "0") +if [ "$SLOT_EXISTS" = "1" ]; then + echo " Replication slot: drasi_building_comfort_slot [OK]" +else + echo " Replication slot: drasi_building_comfort_slot [MISSING]" +fi + +echo +echo "Seeded rooms:" +docker exec building-comfort-postgres psql -U drasi_user -d building_comfort -c \ + "SELECT id, name, temperature, humidity, co2, floor_id FROM \"Room\" ORDER BY id;" + +echo +echo "=== Database setup complete! ===" +echo +echo "Connection details:" +echo " Host: localhost" +echo " Port: ${POSTGRES_HOST_PORT:-5732}" +echo " Database: building_comfort" +echo " User: drasi_user" +echo " Password: drasi_password" diff --git a/tutorials/building-comfort/scripts/start-demo.ps1 b/tutorials/building-comfort/scripts/start-demo.ps1 new file mode 100644 index 0000000..0434fe9 --- /dev/null +++ b/tutorials/building-comfort/scripts/start-demo.ps1 @@ -0,0 +1,42 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start Demo Script (Windows) +# Starts (and seeds) the database, then launches the Streamlit app in the +# foreground. The app embeds the Drasi engine, so this is the only process you +# need. Open the URL it prints (http://localhost:8501 by default) and drive the +# demo from the sidebar. Press Ctrl+C to stop, then run scripts/cleanup.ps1. + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$TutorialDir = Join-Path $ScriptDir ".." + +& (Join-Path $ScriptDir "setup-database.ps1") +if ($LASTEXITCODE -ne 0) { + Write-Host "Database setup failed; not starting the app." + exit 1 +} + +Write-Host "" +Write-Host "Starting the Streamlit app (embeds the Drasi engine)..." +Write-Host "On first run it downloads the Drasi plugins from ghcr.io - give it a moment." +Write-Host "" + +$port = if ($env:STREAMLIT_PORT) { $env:STREAMLIT_PORT } else { "8501" } +Push-Location $TutorialDir +try { + streamlit run app.py --server.port $port --server.address=0.0.0.0 --server.headless=true +} +finally { + Pop-Location +} diff --git a/tutorials/building-comfort/scripts/start-demo.sh b/tutorials/building-comfort/scripts/start-demo.sh new file mode 100755 index 0000000..9c707b0 --- /dev/null +++ b/tutorials/building-comfort/scripts/start-demo.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start Demo Script +# One command to run the whole Building Comfort demo: start (and seed) the +# database, then launch the Streamlit app in the foreground. The app embeds the +# Drasi engine, so this is the only process you need. Open the URL it prints +# (http://localhost:8501 by default) and drive the demo from the sidebar. Press +# Ctrl+C to stop, then run ./scripts/cleanup.sh. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TUTORIAL_DIR="$SCRIPT_DIR/.." + +bash "$SCRIPT_DIR/setup-database.sh" + +echo +echo "Starting the Streamlit app (embeds the Drasi engine)..." +echo "On first run it downloads the Drasi plugins from ghcr.io — give it a moment." +echo + +cd "$TUTORIAL_DIR" +exec streamlit run app.py \ + --server.port "${STREAMLIT_PORT:-8501}" \ + --server.address=0.0.0.0 \ + --server.headless=true diff --git a/tutorials/curbside-pickup/.env.example b/tutorials/curbside-pickup/.env.example new file mode 100644 index 0000000..13ac113 --- /dev/null +++ b/tutorials/curbside-pickup/.env.example @@ -0,0 +1,30 @@ +# Example environment variables for the curbside-pickup tutorial. +# +# Copy this file to .env and adjust as needed: +# cp .env.example .env +# +# The scripts and the app read these with the same defaults shown here. +# +# IMPORTANT: Never commit .env with real credentials! + +# Retail Operations - PostgreSQL (orders) +POSTGRES_HOST=localhost +POSTGRES_PORT=5742 +POSTGRES_HOST_PORT=5742 +POSTGRES_DATABASE=RetailOperations +POSTGRES_USER=drasi_user +POSTGRES_PASSWORD=drasi_password + +# Physical Operations - MySQL (vehicles) +MYSQL_HOST=localhost +MYSQL_PORT=3309 +MYSQL_HOST_PORT=3309 +MYSQL_DATABASE=PhysicalOperations +MYSQL_USER=drasi_user +MYSQL_PASSWORD=drasi_password +# The root password is used by the setup scripts to seed the database and grant +# the replication privileges the Drasi source needs. +MYSQL_ROOT_PASSWORD=root_admin + +# Streamlit UI port. +STREAMLIT_PORT=8501 diff --git a/tutorials/curbside-pickup/.gitignore b/tutorials/curbside-pickup/.gitignore new file mode 100644 index 0000000..1e58996 --- /dev/null +++ b/tutorials/curbside-pickup/.gitignore @@ -0,0 +1,11 @@ +# Local environment overrides +.env + +# Python +__pycache__/ +*.pyc +.venv/ + +# Drasi engine state / logs +data/ +*.log diff --git a/tutorials/curbside-pickup/.streamlit/config.toml b/tutorials/curbside-pickup/.streamlit/config.toml new file mode 100644 index 0000000..b1b7854 --- /dev/null +++ b/tutorials/curbside-pickup/.streamlit/config.toml @@ -0,0 +1,8 @@ +# Streamlit settings for the tutorial. `headless = true` skips the interactive +# first-run email prompt (which would otherwise block startup and leave the port +# unbound), and `address = 0.0.0.0` makes the app reachable through a dev +# container's / Codespaces' forwarded port. +[server] +headless = true +address = "0.0.0.0" +port = 8501 diff --git a/tutorials/curbside-pickup/README.md b/tutorials/curbside-pickup/README.md new file mode 100644 index 0000000..4759d12 --- /dev/null +++ b/tutorials/curbside-pickup/README.md @@ -0,0 +1,472 @@ +<!-- DO NOT EDIT. Generated from _index.md by scripts/render-tutorials.py. Edit _index.md and run `python3 scripts/render-tutorials.py`. --> + +Imagine a store running curbside pickup. The **retail team** manages customer orders in one database and marks an order *ready* when it's prepared. Independently, the **physical operations team** tracks pickup vehicles in *another* database, and a driver sets their location to *Curbside* when they arrive. The two systems never talk to each other. + +You want a single live view with six panels: + +- an **Orders · Preparing** panel and an **Orders · Ready** panel, so an order visibly moves from one to the other the moment it's marked *ready*, +- a **Vehicles · Parking** panel and a **Vehicles · Curbside** panel, so a car moves between them as the driver pulls up, plus the two situations that matter: +- a **Matched Orders** panel that lights up the instant an order is *ready* **and** its driver is at the curbside, so staff know exactly which order to carry out, and +- a **Delayed Orders** panel that flags drivers who have been waiting at the curbside too long while their order still isn't ready. + +The catch: the orders live in **PostgreSQL** and the vehicles live in **MySQL**, and what you care about — a *ready* order whose driver has *arrived* — only exists when you combine a fact from each. Building this the traditional way means **polling both databases**, and you always end up reactive on one side and blind, between polls, on the other. This tutorial builds it with **`drasi-lib`** instead: two sources, six continuous queries (two of which join across both databases), and a **Python reaction** that renders one [Streamlit](https://streamlit.io/) UI. Drasi watches the change feed of *both* databases at once, so a change on **either** side re-evaluates the join immediately — no polling, no blind side. And rather than a separate dashboard plus a separate console, the **one** Streamlit page both shows the six panels *and* has the buttons that drive the changes. + +**Sources** → **Continuous Queries** → **Reactions** + +- **Sources** — Connect to your data sources +- **Continuous Queries** — Define what changes matter +- **Reactions** — Take action automatically + +| Step | What You'll Do | +| ---- | ------------- | +| **[Step 1: Set Up Your Environment](#step-1-of-4-set-up-your-environment)** | Open the dev container (or install Python + Docker locally) | +| **[Step 2: Run the Demo](#step-2-of-4-run-the-demo)** | One command starts both databases and the Streamlit app | +| **[Step 3: Open the UI](#step-3-of-4-open-the-ui)** | Watch all six panels update live | +| **[Step 4: Drive Change](#step-4-of-4-drive-change)** | Use the buttons in the UI to change orders and vehicles, and watch Drasi react | +| **[How It Works](#how-it-works)** | Understand the two sources, the cross-database join, and the six queries | + +> **Before you begin** +> +> - **One process:** the Streamlit app embeds the Drasi engine, so a single command runs everything. It stays in the foreground and you drive changes from your **browser**. +> - **Working directory:** run every command from the tutorial directory (`tutorials/curbside-pickup/`). The dev container opens there automatically; if you're running locally, `cd tutorials/curbside-pickup` first. +> - **Command tabs:** commands are shown in tabs (*bash / zsh* and *PowerShell*). Use the one for your shell. The dev container and Codespaces use *bash*. +> - **Ports:** the Streamlit UI is on `8501`, PostgreSQL is published on `5742`, and MySQL on `3309`. + +## Step 1 of 4: Set Up Your Environment +This tutorial needs **Docker** (it runs PostgreSQL and MySQL) and **Python 3.10+**. The easiest way to get everything is the **dev container**. + +### Option A: Dev Container or GitHub Codespaces (recommended) + +1. Open the [`drasi-python`](https://github.com/drasi-project/drasi-python) repository in VS Code and run **Reopen in Container** (or create a **Codespace** from the repo's **Code** menu). +2. When prompted for a configuration, choose **Curbside Pickup Tutorial (Python)**. +3. Wait for the container to finish. Its setup script installs the tutorial's Python dependencies and the Docker tooling for the two databases. + +That's it. Skip ahead to [Step 2](#step-2-of-4-run-the-demo). + +### Option B: Run Locally + +You'll need **Python 3.10+**, **Docker**, and **bash** for the helper scripts (on Windows use Git Bash or WSL, or run the PowerShell scripts). From the repository root, move into the tutorial directory and install the dependencies: + +**bash / zsh** + +```bash +cd tutorials/curbside-pickup +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +``` + +**PowerShell** + +```powershell +cd tutorials/curbside-pickup +python -m venv .venv; .venv\Scripts\Activate.ps1 +pip install -r requirements.txt +``` + +`drasi-lib` ships as a prebuilt wheel, so no Rust toolchain is needed. It downloads the Postgres and MySQL plugins it needs from `ghcr.io/drasi-project` the first time the app runs. + +## Step 2 of 4: Run the Demo +Start the demo: + +**bash / zsh** + +```bash +bash scripts/start-demo.sh +``` + +**PowerShell** + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/start-demo.ps1 +``` + +The `start-demo` script does two things: + +1. **Starts both databases.** PostgreSQL comes up with logical replication enabled and the `orders` table seeded (three orders, all *preparing*). MySQL comes up with **ROW-based binary logging** and GTID mode enabled and the `vehicles` table seeded (three vehicles, all *Parking*) — the plates match the orders. +2. **Launches the Streamlit app**, which embeds the Drasi engine. + +On first start, the app's embedded engine downloads the plugins it needs (`source/postgres`, `bootstrap/postgres`, `source/mysql`, `bootstrap/mysql`) from `ghcr.io/drasi-project`, connects to both databases, bootstraps the existing rows, starts the six continuous queries, and registers the reaction. Streamlit prints a URL like the following when it's ready: + +```text +Local URL: http://localhost:8501 +``` + +Leave this running. Everything else happens in your **browser**. + +> **Stopping and resetting** +> +> Press **Ctrl+C** in the terminal to stop the app. To remove the database containers when you're completely done, run `bash scripts/cleanup.sh` (bash) or `powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1` (PowerShell). Add `--volumes` (bash) or `-RemoveVolumes` (PowerShell) to also delete the data. + +## Step 3 of 4: Open the UI +The app renders everything from the Python reaction — there's no separate dashboard server and no separate console. **Wait until the app has finished starting** (the first run takes a little longer while the plugins download), then open it in your browser: + +```text +http://localhost:8501 +``` + +In the dev container or Codespaces, port `8501` is forwarded automatically. You'll see the **Curbside Pickup** UI with six panels: + +- 🍕 **Orders · Preparing** and 🍕 **Orders · Ready** — every order in PostgreSQL, split by status. Each row has a button to move it. +- 🚗 **Vehicles · Parking** and 🚗 **Vehicles · Curbside** — every vehicle in MySQL, split by location. Each row has a button to move it. +- 📦 **Matched Orders** — orders that are *ready* whose driver is at the *Curbside* (the `delivery` query). +- ⚠️ **Delayed Orders** — drivers who have waited at the curbside for more than 10 seconds while their order is still being prepared (the `delay` query). + +At bootstrap all three orders are *preparing* and all three vehicles are in *Parking*, so those two panels list every row while their **Ready** and **Curbside** counterparts start empty. The **Matched** and **Delayed** panels also start **empty** and fill in as you drive changes. The **sidebar** has a **Reset** button and a live, color-coded **activity log** of the SQL each button runs. + +## Step 4 of 4: Drive Change +> **No middle tier — the buttons just write to a database** +> +> Each button on an order or vehicle runs a single SQL `UPDATE` — orders in PostgreSQL (via `psycopg`), vehicles in MySQL (via `PyMySQL`) — exactly what a real retail or physical-operations app would run. There's no API to call and no event to publish. Drasi observes the PostgreSQL change through logical replication and the MySQL change through the binary log, re-evaluates the affected queries (including the cross-database join), and calls the Python reaction, which updates the panels. The sidebar's **activity log** shows each statement and which database it hit. + +### Trigger a delivery + +In the 🍕 **Orders · Preparing** panel, press **Mark ready →** on order **1** (Sophia Carter). It immediately leaves *Preparing* and appears in 🍕 **Orders · Ready**. Now, in the 🚗 **Vehicles · Parking** panel, press **To Curbside →** on vehicle **A1234**. + +The vehicle jumps to 🚗 **Vehicles · Curbside**, and within about a second the 📦 **Matched Orders** panel reacts: order **1** appears with its driver and vehicle. Nothing polled anything — Drasi saw the PostgreSQL change and the MySQL change and re-evaluated the cross-database join. Send the vehicle back with **↩ Parking** and the row disappears from **Matched Orders**. + +### Trigger a delay + +Now reproduce the *other* scenario. Pick a vehicle whose order is **not** ready — say **B5678** (Mason Rivera) — and press **To Curbside →**, but **leave order 2 as *preparing***. Nothing happens immediately. After **10 seconds** the ⚠️ **Delayed Orders** panel lights up: order **2** appears, flagging that the driver has been waiting too long. + +This is the interesting one. Drasi doesn't poll to find slow orders — the **continuous query schedules its own future re-evaluation** for the moment the 10-second threshold is crossed, and fires exactly then. If you mark the order *ready* (or send the driver back to *Parking*) before the 10 seconds elapse, the alert never appears. + +### Reset + +Press **Reset everything** in the sidebar to return to the starting state (all orders *preparing*, all vehicles *Parking*). + +## How It Works +The demo is two files in `tutorials/curbside-pickup/`: **`engine.py`** holds the two sources, the six queries, and the `CurbsideEngine` (its Python reaction and the SQL writes); **`app.py`** renders the reaction's state and hosts the controls. The [complete `engine.py`](#putting-it-all-together) is at the end. + +### Two Sources + +The queries join data from two different databases, so the engine adds two sources. Installing the plugins and adding the sources is a few lines of Python: + +```python +for plugin in ("source/postgres", "bootstrap/postgres", "source/mysql", "bootstrap/mysql"): + await drasi.install_plugin(plugin) +await drasi.start() + +# PostgreSQL holds the orders and streams changes via logical replication (CDC). +await drasi.add_source( + "postgres", + "retail-ops", + { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.orders"], + "tableKeys": [{"table": "orders", "keyColumns": ["id"]}], + "slotName": "drasi_curbside_slot", + "publicationName": "drasi_curbside_pub", + }, + bootstrap={"kind": "postgres", **POSTGRES}, +) + +# MySQL holds the vehicles and streams changes via its binary log (binlog). +await drasi.add_source( + "mysql", + "physical-ops", + { + **MYSQL, + "sslMode": "disabled", + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], + }, + bootstrap={ + "kind": "mysql", + **MYSQL, + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], + }, +) +``` + +The MySQL container is started with ROW-based logging, full row images/metadata and GTID mode (see `database/docker-compose.yml`), and `database/mysql-init.sql` grants the Drasi user the replication privileges it needs to read the binlog. Each table row becomes a graph node: `orders` rows become `(o:orders)` and `vehicles` rows become `(v:vehicles)`. + +### The Synthetic Join + +There is no foreign key between the two databases — they're completely separate systems. Drasi creates the relationship in the query with a **synthetic join**, matching a vehicle to an order whenever their `plate` values are equal. The `delivery` and `delay` queries read from **both** sources and declare this join dict in their `joins` list: + +```python +{ + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], +} +``` + +With that declared, the query walks `(o:orders)-[:PICKUP_BY]->(v:vehicles)` as if it were a real graph edge — across two different databases. + +### The Six Continuous Queries + +Four are simple single-source lists, filtered by state, that feed the split **Orders** and **Vehicles** panels — because each matches only part of a table, a row leaves one query's result and joins the other's the instant its status or location changes: + +```cypher +MATCH (o:orders) +WHERE o.status <> 'ready' // orders-ready uses: o.status = 'ready' +RETURN o.id AS id, o.customer_name AS customerName, o.plate AS plate +``` + +The other two join across both databases. **`delivery`** returns an order whenever it is *ready* and its driver's vehicle is at the *Curbside*: + +```cypher +MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) +WHERE o.status = 'ready' AND v.location = 'Curbside' +RETURN o.id AS id, o.customer_name AS customerName, o.plate AS vehicleId, v.make AS vehicleMake +``` + +**`delay`** returns an order whose driver has been at the *Curbside* for more than 10 seconds while the order still isn't *ready*. It uses **`drasi.trueFor`**, which schedules a future re-evaluation and fires the moment the condition has held for the given duration, so the order appears exactly when the 10-second threshold is crossed: + +```cypher +MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) +WHERE o.status <> 'ready' + AND drasi.trueFor(v.location = 'Curbside', duration({ seconds: 10 })) +RETURN o.id AS orderId, o.customer_name AS customerName, + drasi.changeDateTime(v) AS waitingSinceTimestamp +``` + +> **How the timing works** +> +> Both the PostgreSQL and MySQL sources stamp every change with the wall-clock time it happened, which `drasi.changeDateTime()` exposes. `drasi.trueFor` uses that timestamp to anchor its 10-second timer — so the delay query fires exactly when a curbside vehicle has waited long enough — with no extra timestamp columns or application bookkeeping. The `delivery` query uses `drasi.listMax([drasi.changeDateTime(o), drasi.changeDateTime(v)])` to report whichever of the two changes happened later. + +### The Python Reaction + +A single **Python reaction** subscribes to all six queries. Each time a query's result set changes — including the delayed re-evaluation the `delay` query schedules — Drasi calls it with the query id and the diffs, and the callback updates a thread-safe snapshot the UI reads: + +```python +def on_results(event): + query = QUERIES_BY_ID[event["query_id"]] + with lock: + store = snapshot[query.id] # this query's rows, keyed by id + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[query.key], None) + case {"after": dict() as row} | { + "data": dict() as row + }: # ADD / UPDATE / aggregation + store[row[query.key]] = row + + +await drasi.add_python_reaction("ui", [q.id for q in QUERIES], on_results) +``` + +### One integrated Streamlit UI + +The engine runs on its own asyncio event loop in a background daemon thread, created once with `@st.cache_resource`, and `app.py` reads `engine.snapshot()` each run and renders the six panels with a ~1-second auto-refresh. Because it's the same page, the panels can *also* drive the demo: each order and vehicle row has a button that runs the matching `UPDATE`, the sidebar has a **Reset** button, and every write is appended to the activity log the sidebar shows. A `delay` row appearing 10 seconds after you move a vehicle to the curbside is the reaction receiving `drasi.trueFor`'s scheduled re-evaluation — live, with no polling. + +<img src="images/curbside-ui.png" width="960" alt="The Curbside Pickup Streamlit UI: four panels of orders and vehicles with inline buttons across the top, Matched and Delayed panels below, and a sidebar with a reset button and a color-coded SQL activity log"> + +## Putting It All Together +The sections above walked through the Drasi side one piece at a time. Here it is assembled into the single `engine.py` — the two sources, the synthetic join, the six queries, and the `CurbsideEngine` that runs them and writes back. `app.py` (the Streamlit UI) imports `CurbsideEngine` and the query ids from it: + +```python +import asyncio +import os +import threading +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Any + +os.environ.setdefault("RUST_LOG", "warn") # quiet the engine's INFO logging + +import psycopg +import pymysql +from drasi import Drasi + +# --- The two sources --- +POSTGRES = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5742")), + "database": os.environ.get("POSTGRES_DATABASE", "RetailOperations"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), +} +POSTGRES_SOURCE_CONFIG = { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.orders"], + "tableKeys": [{"table": "orders", "keyColumns": ["id"]}], + "slotName": "drasi_curbside_slot", + "publicationName": "drasi_curbside_pub", +} + +MYSQL = { + "host": os.environ.get("MYSQL_HOST", "localhost"), + "port": int(os.environ.get("MYSQL_PORT", "3309")), + "database": os.environ.get("MYSQL_DATABASE", "PhysicalOperations"), + "user": os.environ.get("MYSQL_USER", "drasi_user"), + "password": os.environ.get("MYSQL_PASSWORD", "drasi_password"), +} +MYSQL_SOURCE_CONFIG = { + **MYSQL, + "sslMode": "disabled", # the tutorial container has no TLS + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], +} +# Each source also takes a bootstrap config, and the UI writes through a psycopg +# (Postgres) and a PyMySQL (MySQL) connection built from the dicts above. + +RETAIL_OPS, PHYSICAL_OPS = "retail-ops", "physical-ops" + + +@dataclass(frozen=True) +class Query: + id: str + key: str # the RETURN column that identifies each row + sources: list[str] + cypher: str + joins: list[dict] = field(default_factory=list) + + +QUERIES = [ + # Four single-source list queries split orders and vehicles by state. Each has + # the same shape; orders-preparing is shown here, and orders-ready / + # vehicles-parking / vehicles-curbside flip the WHERE clause. + Query( + id="orders-preparing", + key="id", + sources=[RETAIL_OPS], + cypher="MATCH (o:orders) WHERE o.status <> 'ready' " + "RETURN o.id AS id, o.customer_name AS customerName, o.plate AS plate", + ), + # delivery: an order is ready AND its driver's vehicle is at the curbside. The + # synthetic join matches a vehicle to an order by equal plate, across databases. + Query( + id="delivery", + key="id", + sources=[RETAIL_OPS, PHYSICAL_OPS], + joins=[ + { + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], + } + ], + cypher=""" + MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) + WHERE o.status = 'ready' AND v.location = 'Curbside' + RETURN o.id AS id, o.customer_name AS customerName, o.plate AS vehicleId + """, + ), + # delay: at the curbside over 10s while the order is still not ready. + Query( + id="delay", + key="orderId", + sources=[RETAIL_OPS, PHYSICAL_OPS], + joins=[ + { + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], + } + ], + cypher=""" + MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) + WHERE o.status <> 'ready' + AND drasi.trueFor(v.location = 'Curbside', duration({ seconds: 10 })) + RETURN o.id AS orderId, o.customer_name AS customerName, + drasi.changeDateTime(v) AS waitingSinceTimestamp + """, + ), +] + + +class CurbsideEngine: + """Runs Drasi over two databases and exposes a thread-safe snapshot.""" + + def __init__(self): + self._lock = threading.Lock() + self._results = {} # query id -> {row key -> row} + self._key_by_id = {q.id: q.key for q in QUERIES} + self._activity = deque(maxlen=20) # recent SQL writes for the sidebar + self._ready = threading.Event() + self._drasi = None + # Run the engine on its own event loop in a background thread. + threading.Thread(target=self._run, daemon=True).start() + + def _run(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._start()) + self._ready.set() + loop.run_forever() + + async def _start(self): + self._drasi = await Drasi.create("curbside-pickup") + for plugin in ("source/postgres", "bootstrap/postgres", "source/mysql", "bootstrap/mysql"): + await self._drasi.install_plugin(plugin) + await self._drasi.start() + + await self._drasi.add_source("postgres", RETAIL_OPS, POSTGRES_SOURCE_CONFIG, bootstrap=...) + await self._drasi.add_source("mysql", PHYSICAL_OPS, MYSQL_SOURCE_CONFIG, bootstrap=...) + for q in QUERIES: + await self._drasi.add_query(q.id, q.cypher, q.sources, joins=q.joins or None) + for q in QUERIES: + await self._drasi.wait_for_query(q.id) + rows = await self._drasi.get_query_results(q.id) + self._results[q.id] = {r[q.key]: r for r in rows} + + await self._drasi.add_python_reaction("ui", [q.id for q in QUERIES], self._on_results) + + def _on_results(self, event): + key = self._key_by_id[event["query_id"]] + with self._lock: + store = self._results.setdefault(event["query_id"], {}) + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[key], None) + case {"after": dict() as row} | {"data": dict() as row}: + store[row[key]] = row + + def snapshot(self): + with self._lock: + return { + "results": {q: list(s.values()) for q, s in self._results.items()}, + "activity": list(self._activity), + } + + # set_order_status(id, status) writes to Postgres, set_vehicle_location(plate, + # location) writes to MySQL, and reset() returns everything to the start. + # Each records the SQL it ran in the activity log. +``` + +That's the whole Drasi side in one file: two databases joined by plate, six continuous queries (one temporal), and a reaction that keeps a snapshot the UI renders and drives. + +## Clean Up +When you're finished, stop the app with **Ctrl+C** in the terminal, then remove the database containers: + +**bash / zsh** + +```bash +# Stop containers, keep data +bash scripts/cleanup.sh + +# Stop containers and delete the data volumes +bash scripts/cleanup.sh --volumes +``` + +**PowerShell** + +```powershell +# Stop containers, keep data +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 + +# Stop containers and delete the data volumes +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 -RemoveVolumes +``` + +## What You Learned +- **Sources** connect Drasi to live data — here, **two** databases at once, PostgreSQL via logical replication and MySQL via the binary log. +- **Continuous Queries** with a **synthetic join** relate rows across those two databases (order ↔ vehicle by plate), and a **temporal** query (`drasi.trueFor`) fires exactly when a condition has held long enough — no polling, no bookkeeping. +- A **Python reaction** turned all six queries into one live UI, and because it's a Streamlit page, the same UI hosts the controls that drive the changes — no dashboard reaction and no separate console. +- Because Drasi emits only what *changed*, every panel updates the instant the data does, whichever database changed. + +From here, try changing the delay threshold, adding an order/vehicle, or pointing the sources at your own databases. diff --git a/tutorials/curbside-pickup/_index.md b/tutorials/curbside-pickup/_index.md new file mode 100644 index 0000000..9207e3a --- /dev/null +++ b/tutorials/curbside-pickup/_index.md @@ -0,0 +1,506 @@ +--- +type: "docs" +title: "Curbside Pickup" +linkTitle: "Curbside Pickup" +weight: 40 +description: > + Join a PostgreSQL orders store and a MySQL vehicles store by license plate, and render the result live in one Streamlit UI driven by a Python reaction — with the controls that drive the changes built right in. +--- + +Imagine a store running curbside pickup. The **retail team** manages customer orders in one database and marks an order *ready* when it's prepared. Independently, the **physical operations team** tracks pickup vehicles in *another* database, and a driver sets their location to *Curbside* when they arrive. The two systems never talk to each other. + +You want a single live view with six panels: + +- an **Orders · Preparing** panel and an **Orders · Ready** panel, so an order visibly moves from one to the other the moment it's marked *ready*, +- a **Vehicles · Parking** panel and a **Vehicles · Curbside** panel, so a car moves between them as the driver pulls up, plus the two situations that matter: +- a **Matched Orders** panel that lights up the instant an order is *ready* **and** its driver is at the curbside, so staff know exactly which order to carry out, and +- a **Delayed Orders** panel that flags drivers who have been waiting at the curbside too long while their order still isn't ready. + +The catch: the orders live in **PostgreSQL** and the vehicles live in **MySQL**, and what you care about — a *ready* order whose driver has *arrived* — only exists when you combine a fact from each. Building this the traditional way means **polling both databases**, and you always end up reactive on one side and blind, between polls, on the other. This tutorial builds it with **`drasi-lib`** instead: two sources, six continuous queries (two of which join across both databases), and a **Python reaction** that renders one [Streamlit](https://streamlit.io/) UI. Drasi watches the change feed of *both* databases at once, so a change on **either** side re-evaluates the join immediately — no polling, no blind side. And rather than a separate dashboard plus a separate console, the **one** Streamlit page both shows the six panels *and* has the buttons that drive the changes. + +<div class="flow-diagram"> + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-database"></i> + </div> + <div class="flow-step__label">Sources</div> + <div class="flow-step__description">Connect to your data sources</div> + </div> + + <div class="flow-arrow"> + <i class="fas fa-arrow-right"></i> + </div> + + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-filter"></i> + </div> + <div class="flow-step__label">Continuous Queries</div> + <div class="flow-step__description">Define what changes matter</div> + </div> + + <div class="flow-arrow"> + <i class="fas fa-arrow-right"></i> + </div> + + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-bolt"></i> + </div> + <div class="flow-step__label">Reactions</div> + <div class="flow-step__description">Take action automatically</div> + </div> +</div> + +| Step | What You'll Do | +| ---- | ------------- | +| **[Step 1: Set Up Your Environment](#setup)** | Open the dev container (or install Python + Docker locally) | +| **[Step 2: Run the Demo](#run)** | One command starts both databases and the Streamlit app | +| **[Step 3: Open the UI](#ui)** | Watch all six panels update live | +| **[Step 4: Drive Change](#drive)** | Use the buttons in the UI to change orders and vehicles, and watch Drasi react | +| **[How It Works](#how)** | Understand the two sources, the cross-database join, and the six queries | + +{{% alert title="Before you begin" color="info" %}} +- **One process:** the Streamlit app embeds the Drasi engine, so a single command runs everything. It stays in the foreground and you drive changes from your **browser**. +- **Working directory:** run every command from the tutorial directory (`tutorials/curbside-pickup/`). The dev container opens there automatically; if you're running locally, `cd tutorials/curbside-pickup` first. +- **Command tabs:** commands are shown in tabs (*bash / zsh* and *PowerShell*). Use the one for your shell. The dev container and Codespaces use *bash*. +- **Ports:** the Streamlit UI is on `8501`, PostgreSQL is published on `5742`, and MySQL on `3309`. +{{% /alert %}} + +## Step 1 of 4: Set Up Your Environment {#setup} + +This tutorial needs **Docker** (it runs PostgreSQL and MySQL) and **Python 3.10+**. The easiest way to get everything is the **dev container**. + +### Option A: Dev Container or GitHub Codespaces (recommended) + +1. Open the [`drasi-python`](https://github.com/drasi-project/drasi-python) repository in VS Code and run **Reopen in Container** (or create a **Codespace** from the repo's **Code** menu). +2. When prompted for a configuration, choose **Curbside Pickup Tutorial (Python)**. +3. Wait for the container to finish. Its setup script installs the tutorial's Python dependencies and the Docker tooling for the two databases. + +That's it. Skip ahead to [Step 2](#run). + +### Option B: Run Locally + +You'll need **Python 3.10+**, **Docker**, and **bash** for the helper scripts (on Windows use Git Bash or WSL, or run the PowerShell scripts). From the repository root, move into the tutorial directory and install the dependencies: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +cd tutorials/curbside-pickup +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +cd tutorials/curbside-pickup +python -m venv .venv; .venv\Scripts\Activate.ps1 +pip install -r requirements.txt +{{< /tab >}} +{{< /tabpane >}} + +`drasi-lib` ships as a prebuilt wheel, so no Rust toolchain is needed. It downloads the Postgres and MySQL plugins it needs from `ghcr.io/drasi-project` the first time the app runs. + +## Step 2 of 4: Run the Demo {#run} + +Start the demo: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +bash scripts/start-demo.sh +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +powershell -ExecutionPolicy Bypass -File scripts/start-demo.ps1 +{{< /tab >}} +{{< /tabpane >}} + +The `start-demo` script does two things: + +1. **Starts both databases.** PostgreSQL comes up with logical replication enabled and the `orders` table seeded (three orders, all *preparing*). MySQL comes up with **ROW-based binary logging** and GTID mode enabled and the `vehicles` table seeded (three vehicles, all *Parking*) — the plates match the orders. +2. **Launches the Streamlit app**, which embeds the Drasi engine. + +On first start, the app's embedded engine downloads the plugins it needs (`source/postgres`, `bootstrap/postgres`, `source/mysql`, `bootstrap/mysql`) from `ghcr.io/drasi-project`, connects to both databases, bootstraps the existing rows, starts the six continuous queries, and registers the reaction. Streamlit prints a URL like the following when it's ready: + +```text +Local URL: http://localhost:8501 +``` + +Leave this running. Everything else happens in your **browser**. + +{{% alert title="Stopping and resetting" color="info" %}} +Press **Ctrl+C** in the terminal to stop the app. To remove the database containers when you're completely done, run `bash scripts/cleanup.sh` (bash) or `powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1` (PowerShell). Add `--volumes` (bash) or `-RemoveVolumes` (PowerShell) to also delete the data. +{{% /alert %}} + +## Step 3 of 4: Open the UI {#ui} + +The app renders everything from the Python reaction — there's no separate dashboard server and no separate console. **Wait until the app has finished starting** (the first run takes a little longer while the plugins download), then open it in your browser: + +```text +http://localhost:8501 +``` + +In the dev container or Codespaces, port `8501` is forwarded automatically. You'll see the **Curbside Pickup** UI with six panels: + +- 🍕 **Orders · Preparing** and 🍕 **Orders · Ready** — every order in PostgreSQL, split by status. Each row has a button to move it. +- 🚗 **Vehicles · Parking** and 🚗 **Vehicles · Curbside** — every vehicle in MySQL, split by location. Each row has a button to move it. +- 📦 **Matched Orders** — orders that are *ready* whose driver is at the *Curbside* (the `delivery` query). +- ⚠️ **Delayed Orders** — drivers who have waited at the curbside for more than 10 seconds while their order is still being prepared (the `delay` query). + +At bootstrap all three orders are *preparing* and all three vehicles are in *Parking*, so those two panels list every row while their **Ready** and **Curbside** counterparts start empty. The **Matched** and **Delayed** panels also start **empty** and fill in as you drive changes. The **sidebar** has a **Reset** button and a live, color-coded **activity log** of the SQL each button runs. + +## Step 4 of 4: Drive Change {#drive} + +{{% alert title="No middle tier — the buttons just write to a database" color="info" %}} +Each button on an order or vehicle runs a single SQL `UPDATE` — orders in PostgreSQL (via `psycopg`), vehicles in MySQL (via `PyMySQL`) — exactly what a real retail or physical-operations app would run. There's no API to call and no event to publish. Drasi observes the PostgreSQL change through logical replication and the MySQL change through the binary log, re-evaluates the affected queries (including the cross-database join), and calls the Python reaction, which updates the panels. The sidebar's **activity log** shows each statement and which database it hit. +{{% /alert %}} + +### Trigger a delivery + +In the 🍕 **Orders · Preparing** panel, press **Mark ready →** on order **1** (Sophia Carter). It immediately leaves *Preparing* and appears in 🍕 **Orders · Ready**. Now, in the 🚗 **Vehicles · Parking** panel, press **To Curbside →** on vehicle **A1234**. + +The vehicle jumps to 🚗 **Vehicles · Curbside**, and within about a second the 📦 **Matched Orders** panel reacts: order **1** appears with its driver and vehicle. Nothing polled anything — Drasi saw the PostgreSQL change and the MySQL change and re-evaluated the cross-database join. Send the vehicle back with **↩ Parking** and the row disappears from **Matched Orders**. + +### Trigger a delay + +Now reproduce the *other* scenario. Pick a vehicle whose order is **not** ready — say **B5678** (Mason Rivera) — and press **To Curbside →**, but **leave order 2 as *preparing***. Nothing happens immediately. After **10 seconds** the ⚠️ **Delayed Orders** panel lights up: order **2** appears, flagging that the driver has been waiting too long. + +This is the interesting one. Drasi doesn't poll to find slow orders — the **continuous query schedules its own future re-evaluation** for the moment the 10-second threshold is crossed, and fires exactly then. If you mark the order *ready* (or send the driver back to *Parking*) before the 10 seconds elapse, the alert never appears. + +### Reset + +Press **Reset everything** in the sidebar to return to the starting state (all orders *preparing*, all vehicles *Parking*). + +## How It Works {#how} + +The demo is two files in `tutorials/curbside-pickup/`: **`engine.py`** holds the two sources, the six queries, and the `CurbsideEngine` (its Python reaction and the SQL writes); **`app.py`** renders the reaction's state and hosts the controls. The [complete `engine.py`](#all) is at the end. + +### Two Sources + +The queries join data from two different databases, so the engine adds two sources. Installing the plugins and adding the sources is a few lines of Python: + +```python +for plugin in ("source/postgres", "bootstrap/postgres", "source/mysql", "bootstrap/mysql"): + await drasi.install_plugin(plugin) +await drasi.start() + +# PostgreSQL holds the orders and streams changes via logical replication (CDC). +await drasi.add_source( + "postgres", + "retail-ops", + { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.orders"], + "tableKeys": [{"table": "orders", "keyColumns": ["id"]}], + "slotName": "drasi_curbside_slot", + "publicationName": "drasi_curbside_pub", + }, + bootstrap={"kind": "postgres", **POSTGRES}, +) + +# MySQL holds the vehicles and streams changes via its binary log (binlog). +await drasi.add_source( + "mysql", + "physical-ops", + { + **MYSQL, + "sslMode": "disabled", + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], + }, + bootstrap={ + "kind": "mysql", + **MYSQL, + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], + }, +) +``` + +The MySQL container is started with ROW-based logging, full row images/metadata and GTID mode (see `database/docker-compose.yml`), and `database/mysql-init.sql` grants the Drasi user the replication privileges it needs to read the binlog. Each table row becomes a graph node: `orders` rows become `(o:orders)` and `vehicles` rows become `(v:vehicles)`. + +### The Synthetic Join + +There is no foreign key between the two databases — they're completely separate systems. Drasi creates the relationship in the query with a **synthetic join**, matching a vehicle to an order whenever their `plate` values are equal. The `delivery` and `delay` queries read from **both** sources and declare this join dict in their `joins` list: + +```python +{ + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], +} +``` + +With that declared, the query walks `(o:orders)-[:PICKUP_BY]->(v:vehicles)` as if it were a real graph edge — across two different databases. + +### The Six Continuous Queries + +Four are simple single-source lists, filtered by state, that feed the split **Orders** and **Vehicles** panels — because each matches only part of a table, a row leaves one query's result and joins the other's the instant its status or location changes: + +```cypher +MATCH (o:orders) +WHERE o.status <> 'ready' // orders-ready uses: o.status = 'ready' +RETURN o.id AS id, o.customer_name AS customerName, o.plate AS plate +``` + +The other two join across both databases. **`delivery`** returns an order whenever it is *ready* and its driver's vehicle is at the *Curbside*: + +```cypher +MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) +WHERE o.status = 'ready' AND v.location = 'Curbside' +RETURN o.id AS id, o.customer_name AS customerName, o.plate AS vehicleId, v.make AS vehicleMake +``` + +**`delay`** returns an order whose driver has been at the *Curbside* for more than 10 seconds while the order still isn't *ready*. It uses **`drasi.trueFor`**, which schedules a future re-evaluation and fires the moment the condition has held for the given duration, so the order appears exactly when the 10-second threshold is crossed: + +```cypher +MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) +WHERE o.status <> 'ready' + AND drasi.trueFor(v.location = 'Curbside', duration({ seconds: 10 })) +RETURN o.id AS orderId, o.customer_name AS customerName, + drasi.changeDateTime(v) AS waitingSinceTimestamp +``` + +{{% alert title="How the timing works" color="info" %}} +Both the PostgreSQL and MySQL sources stamp every change with the wall-clock time it happened, which `drasi.changeDateTime()` exposes. `drasi.trueFor` uses that timestamp to anchor its 10-second timer — so the delay query fires exactly when a curbside vehicle has waited long enough — with no extra timestamp columns or application bookkeeping. The `delivery` query uses `drasi.listMax([drasi.changeDateTime(o), drasi.changeDateTime(v)])` to report whichever of the two changes happened later. +{{% /alert %}} + +### The Python Reaction + +A single **Python reaction** subscribes to all six queries. Each time a query's result set changes — including the delayed re-evaluation the `delay` query schedules — Drasi calls it with the query id and the diffs, and the callback updates a thread-safe snapshot the UI reads: + +```python +def on_results(event): + query = QUERIES_BY_ID[event["query_id"]] + with lock: + store = snapshot[query.id] # this query's rows, keyed by id + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[query.key], None) + case {"after": dict() as row} | { + "data": dict() as row + }: # ADD / UPDATE / aggregation + store[row[query.key]] = row + + +await drasi.add_python_reaction("ui", [q.id for q in QUERIES], on_results) +``` + +### One integrated Streamlit UI + +The engine runs on its own asyncio event loop in a background daemon thread, created once with `@st.cache_resource`, and `app.py` reads `engine.snapshot()` each run and renders the six panels with a ~1-second auto-refresh. Because it's the same page, the panels can *also* drive the demo: each order and vehicle row has a button that runs the matching `UPDATE`, the sidebar has a **Reset** button, and every write is appended to the activity log the sidebar shows. A `delay` row appearing 10 seconds after you move a vehicle to the curbside is the reaction receiving `drasi.trueFor`'s scheduled re-evaluation — live, with no polling. + +<img src="images/curbside-ui.png" width="960" alt="The Curbside Pickup Streamlit UI: four panels of orders and vehicles with inline buttons across the top, Matched and Delayed panels below, and a sidebar with a reset button and a color-coded SQL activity log"> + +## Putting It All Together {#all} + +The sections above walked through the Drasi side one piece at a time. Here it is assembled into the single `engine.py` — the two sources, the synthetic join, the six queries, and the `CurbsideEngine` that runs them and writes back. `app.py` (the Streamlit UI) imports `CurbsideEngine` and the query ids from it: + +```python +import asyncio +import os +import threading +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Any + +os.environ.setdefault("RUST_LOG", "warn") # quiet the engine's INFO logging + +import psycopg +import pymysql +from drasi import Drasi + +# --- The two sources --- +POSTGRES = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5742")), + "database": os.environ.get("POSTGRES_DATABASE", "RetailOperations"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), +} +POSTGRES_SOURCE_CONFIG = { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.orders"], + "tableKeys": [{"table": "orders", "keyColumns": ["id"]}], + "slotName": "drasi_curbside_slot", + "publicationName": "drasi_curbside_pub", +} + +MYSQL = { + "host": os.environ.get("MYSQL_HOST", "localhost"), + "port": int(os.environ.get("MYSQL_PORT", "3309")), + "database": os.environ.get("MYSQL_DATABASE", "PhysicalOperations"), + "user": os.environ.get("MYSQL_USER", "drasi_user"), + "password": os.environ.get("MYSQL_PASSWORD", "drasi_password"), +} +MYSQL_SOURCE_CONFIG = { + **MYSQL, + "sslMode": "disabled", # the tutorial container has no TLS + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], +} +# Each source also takes a bootstrap config, and the UI writes through a psycopg +# (Postgres) and a PyMySQL (MySQL) connection built from the dicts above. + +RETAIL_OPS, PHYSICAL_OPS = "retail-ops", "physical-ops" + + +@dataclass(frozen=True) +class Query: + id: str + key: str # the RETURN column that identifies each row + sources: list[str] + cypher: str + joins: list[dict] = field(default_factory=list) + + +QUERIES = [ + # Four single-source list queries split orders and vehicles by state. Each has + # the same shape; orders-preparing is shown here, and orders-ready / + # vehicles-parking / vehicles-curbside flip the WHERE clause. + Query( + id="orders-preparing", + key="id", + sources=[RETAIL_OPS], + cypher="MATCH (o:orders) WHERE o.status <> 'ready' " + "RETURN o.id AS id, o.customer_name AS customerName, o.plate AS plate", + ), + # delivery: an order is ready AND its driver's vehicle is at the curbside. The + # synthetic join matches a vehicle to an order by equal plate, across databases. + Query( + id="delivery", + key="id", + sources=[RETAIL_OPS, PHYSICAL_OPS], + joins=[ + { + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], + } + ], + cypher=""" + MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) + WHERE o.status = 'ready' AND v.location = 'Curbside' + RETURN o.id AS id, o.customer_name AS customerName, o.plate AS vehicleId + """, + ), + # delay: at the curbside over 10s while the order is still not ready. + Query( + id="delay", + key="orderId", + sources=[RETAIL_OPS, PHYSICAL_OPS], + joins=[ + { + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], + } + ], + cypher=""" + MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) + WHERE o.status <> 'ready' + AND drasi.trueFor(v.location = 'Curbside', duration({ seconds: 10 })) + RETURN o.id AS orderId, o.customer_name AS customerName, + drasi.changeDateTime(v) AS waitingSinceTimestamp + """, + ), +] + + +class CurbsideEngine: + """Runs Drasi over two databases and exposes a thread-safe snapshot.""" + + def __init__(self): + self._lock = threading.Lock() + self._results = {} # query id -> {row key -> row} + self._key_by_id = {q.id: q.key for q in QUERIES} + self._activity = deque(maxlen=20) # recent SQL writes for the sidebar + self._ready = threading.Event() + self._drasi = None + # Run the engine on its own event loop in a background thread. + threading.Thread(target=self._run, daemon=True).start() + + def _run(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._start()) + self._ready.set() + loop.run_forever() + + async def _start(self): + self._drasi = await Drasi.create("curbside-pickup") + for plugin in ("source/postgres", "bootstrap/postgres", "source/mysql", "bootstrap/mysql"): + await self._drasi.install_plugin(plugin) + await self._drasi.start() + + await self._drasi.add_source("postgres", RETAIL_OPS, POSTGRES_SOURCE_CONFIG, bootstrap=...) + await self._drasi.add_source("mysql", PHYSICAL_OPS, MYSQL_SOURCE_CONFIG, bootstrap=...) + for q in QUERIES: + await self._drasi.add_query(q.id, q.cypher, q.sources, joins=q.joins or None) + for q in QUERIES: + await self._drasi.wait_for_query(q.id) + rows = await self._drasi.get_query_results(q.id) + self._results[q.id] = {r[q.key]: r for r in rows} + + await self._drasi.add_python_reaction("ui", [q.id for q in QUERIES], self._on_results) + + def _on_results(self, event): + key = self._key_by_id[event["query_id"]] + with self._lock: + store = self._results.setdefault(event["query_id"], {}) + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[key], None) + case {"after": dict() as row} | {"data": dict() as row}: + store[row[key]] = row + + def snapshot(self): + with self._lock: + return { + "results": {q: list(s.values()) for q, s in self._results.items()}, + "activity": list(self._activity), + } + + # set_order_status(id, status) writes to Postgres, set_vehicle_location(plate, + # location) writes to MySQL, and reset() returns everything to the start. + # Each records the SQL it ran in the activity log. +``` + +That's the whole Drasi side in one file: two databases joined by plate, six continuous queries (one temporal), and a reaction that keeps a snapshot the UI renders and drives. + +## Clean Up {#cleanup} + +When you're finished, stop the app with **Ctrl+C** in the terminal, then remove the database containers: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +# Stop containers, keep data +bash scripts/cleanup.sh + +# Stop containers and delete the data volumes +bash scripts/cleanup.sh --volumes +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +# Stop containers, keep data +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 + +# Stop containers and delete the data volumes +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 -RemoveVolumes +{{< /tab >}} +{{< /tabpane >}} + +## What You Learned {#summary} + +- **Sources** connect Drasi to live data — here, **two** databases at once, PostgreSQL via logical replication and MySQL via the binary log. +- **Continuous Queries** with a **synthetic join** relate rows across those two databases (order ↔ vehicle by plate), and a **temporal** query (`drasi.trueFor`) fires exactly when a condition has held long enough — no polling, no bookkeeping. +- A **Python reaction** turned all six queries into one live UI, and because it's a Streamlit page, the same UI hosts the controls that drive the changes — no dashboard reaction and no separate console. +- Because Drasi emits only what *changed*, every panel updates the instant the data does, whichever database changed. + +From here, try changing the delay threshold, adding an order/vehicle, or pointing the sources at your own databases. diff --git a/tutorials/curbside-pickup/app.py b/tutorials/curbside-pickup/app.py new file mode 100644 index 0000000..8424cf1 --- /dev/null +++ b/tutorials/curbside-pickup/app.py @@ -0,0 +1,188 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Curbside Pickup -- one integrated Streamlit UI driven by a Drasi reaction. + +Run it with: + + streamlit run app.py + +The six panels are rendered live from a Python reaction over two databases, and +the same page drives the changes: the Orders and Vehicles panels have inline +buttons that write to Postgres / MySQL, and the sidebar has a reset button and a +color-coded log of the SQL each button ran. No dashboard reaction, no separate +console -- the reaction *is* the application. +""" + +from __future__ import annotations + +from typing import Any + +import streamlit as st +from engine import ( + DELAY, + DELAY_SECONDS, + DELIVERY, + ORDERS_PREPARING, + ORDERS_READY, + VEHICLES_CURBSIDE, + VEHICLES_PARKING, + CurbsideEngine, +) +from streamlit_autorefresh import st_autorefresh + +st.set_page_config(page_title="Curbside Pickup", page_icon="🛒", layout="wide") + + +@st.cache_resource +def get_engine() -> CurbsideEngine: + """Create (once) and return the shared Drasi engine.""" + engine = CurbsideEngine() + with st.spinner("Starting Drasi: installing plugins and connecting to Postgres + MySQL..."): + engine.wait_ready() + return engine + + +def order_panel(rows: list[dict[str, Any]], *, ready: bool, engine: CurbsideEngine) -> None: + title = "🍕 Orders · Ready" if ready else "🍕 Orders · Preparing" + st.subheader(title) + if not rows: + st.caption("_No orders ready._" if ready else "_No orders being prepared._") + return + for row in sorted(rows, key=lambda r: r["orderId"]): + st.markdown(f"**Order {row['orderId']}** — {row['customerName']} — plate `{row['plate']}`") + if ready: + if st.button("↩ Preparing", key=f"order-prep-{row['id']}", use_container_width=True): + engine.set_order_status(int(row["id"]), "preparing") + st.rerun() + else: + if st.button("Mark ready →", key=f"order-ready-{row['id']}", use_container_width=True): + engine.set_order_status(int(row["id"]), "ready") + st.rerun() + + +def vehicle_panel(rows: list[dict[str, Any]], *, curbside: bool, engine: CurbsideEngine) -> None: + title = "🚗 Vehicles · Curbside" if curbside else "🚗 Vehicles · Parking" + st.subheader(title) + if not rows: + st.caption("_No vehicles at the curb._" if curbside else "_No vehicles parked._") + return + for row in sorted(rows, key=lambda r: r["plate"]): + st.markdown(f"**`{row['plate']}`** — {row['color']} {row['make']} {row['model']}") + if curbside: + if st.button("↩ Parking", key=f"veh-park-{row['plate']}", use_container_width=True): + engine.set_vehicle_location(row["plate"], "Parking") + st.rerun() + else: + if st.button("To Curbside →", key=f"veh-curb-{row['plate']}", use_container_width=True): + engine.set_vehicle_location(row["plate"], "Curbside") + st.rerun() + + +def matched_panel(rows: list[dict[str, Any]]) -> None: + st.subheader("📦 Matched Orders") + st.caption("Ready **and** the driver is at the curbside — carry it out.") + if not rows: + st.info("No matched orders yet. Mark an order ready and bring its vehicle to the curbside.") + return + for row in sorted(rows, key=lambda r: r["orderId"]): + st.success( + f"**Order {row['orderId']}** — {row['customerName']} — " + f"{row['vehicleColor']} {row['vehicleMake']} {row['vehicleModel']} " + f"(`{row['vehicleId']}`) is at the curbside." + ) + + +def _short_time(value: Any) -> str: + """Show just the clock time from an ISO timestamp like 2026-07-31T00:58:53+00:00.""" + text = str(value) + if "T" in text: + return text.split("T", 1)[1][:8] + return text + + +def delayed_panel(rows: list[dict[str, Any]]) -> None: + st.subheader("⚠️ Delayed Orders") + st.caption(f"At the curbside over {DELAY_SECONDS}s while the order is still not ready.") + if not rows: + st.info("No delayed orders.") + return + for row in sorted(rows, key=lambda r: r["orderId"]): + st.warning( + f"**Order {row['orderId']}** — {row['customerName']} — " + f"waiting since {_short_time(row.get('waitingSinceTimestamp', ''))}" + ) + + +def render_sidebar(engine: CurbsideEngine, activity: list[dict[str, Any]]) -> None: + st.sidebar.header("Drive the demo") + st.sidebar.caption( + "The buttons on each order and vehicle run a real SQL UPDATE — orders in " + "PostgreSQL, vehicles in MySQL. Drasi observes both via CDC and the panels " + "update on their own." + ) + if st.sidebar.button("Reset everything", use_container_width=True): + engine.reset() + st.toast("Reset: all orders preparing, all vehicles parking.") + st.rerun() + + st.sidebar.divider() + st.sidebar.subheader("Activity log") + if not activity: + st.sidebar.caption("SQL you run will appear here, colored by database.") + return + for entry in reversed(activity): + color = "blue" if entry["db"] == "PostgreSQL" else "orange" + st.sidebar.markdown( + f":gray[{entry['at']}] :{color}[**[{entry['db']}]**] \n`{entry['sql']}`" + ) + + +def main() -> None: + engine = get_engine() + st_autorefresh(interval=1000, key="curbside-refresh") + + snapshot = engine.snapshot() + if snapshot["error"]: + st.error(f"The Drasi engine failed: {snapshot['error']}") + return + results = snapshot["results"] + + st.title("🛒 Curbside Pickup") + st.caption( + "Orders live in **PostgreSQL**, vehicles live in **MySQL**. Drasi joins them " + "by license plate and reacts to changes on either side — no polling." + ) + + render_sidebar(engine, snapshot["activity"]) + + top = st.columns(4) + with top[0], st.container(border=True): + order_panel(results.get(ORDERS_PREPARING, []), ready=False, engine=engine) + with top[1], st.container(border=True): + order_panel(results.get(ORDERS_READY, []), ready=True, engine=engine) + with top[2], st.container(border=True): + vehicle_panel(results.get(VEHICLES_PARKING, []), curbside=False, engine=engine) + with top[3], st.container(border=True): + vehicle_panel(results.get(VEHICLES_CURBSIDE, []), curbside=True, engine=engine) + + bottom = st.columns(2) + with bottom[0], st.container(border=True): + matched_panel(results.get(DELIVERY, [])) + with bottom[1], st.container(border=True): + delayed_panel(results.get(DELAY, [])) + + +if __name__ == "__main__": + main() diff --git a/tutorials/curbside-pickup/database/docker-compose.yml b/tutorials/curbside-pickup/database/docker-compose.yml new file mode 100644 index 0000000..20322d5 --- /dev/null +++ b/tutorials/curbside-pickup/database/docker-compose.yml @@ -0,0 +1,93 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Two independent databases for the Curbside Pickup tutorial: +# - postgres : the Retail Operations store (orders) +# - mysql : the Physical Operations store (vehicles) +# Both are configured for change data capture (CDC) so Drasi can stream their +# changes in real time. + +name: curbside-pickup + +services: + # --------------------------------------------------------------------------- + # Retail Operations - PostgreSQL (orders) + # --------------------------------------------------------------------------- + postgres: + image: postgres:15-alpine + container_name: curbside-pickup-postgres + environment: + POSTGRES_DB: RetailOperations + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres_admin + command: + # wal_level=logical is required for logical replication (CDC). + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_replication_slots=10" + - "-c" + - "max_wal_senders=10" + ports: + - "${POSTGRES_HOST_PORT:-5742}:5432" + volumes: + - curbside_postgres_data:/var/lib/postgresql/data + healthcheck: + # Probe over TCP (-h localhost), not the Unix socket. On first init the + # postgres image runs a temporary socket-only server to create POSTGRES_DB; + # a socket probe can report "ready" before RetailOperations exists. + test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d RetailOperations"] + interval: 5s + timeout: 5s + retries: 5 + + # --------------------------------------------------------------------------- + # Physical Operations - MySQL (vehicles) + # --------------------------------------------------------------------------- + # The Drasi MySQL source captures changes from the binary log (binlog), so the + # server is started with ROW-based binary logging, full row images, full row + # metadata (column names) and GTID mode - all required by the source. The + # drasi_user is granted REPLICATION SLAVE / REPLICATION CLIENT in + # database/mysql-init.sql so it can stream the binlog. + mysql: + image: mysql:8.0 + container_name: curbside-pickup-mysql + environment: + MYSQL_ROOT_PASSWORD: "${MYSQL_ROOT_PASSWORD:-root_admin}" + MYSQL_DATABASE: "${MYSQL_DATABASE:-PhysicalOperations}" + MYSQL_USER: "${MYSQL_USER:-drasi_user}" + MYSQL_PASSWORD: "${MYSQL_PASSWORD:-drasi_password}" + command: + - "--server-id=1" + - "--log-bin=mysql-bin" + - "--binlog-format=ROW" + - "--binlog-row-image=FULL" + - "--binlog-row-metadata=FULL" + - "--gtid-mode=ON" + - "--enforce-gtid-consistency=ON" + ports: + - "${MYSQL_HOST_PORT:-3309}:3306" + volumes: + - curbside_mysql_data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -p\"$$MYSQL_ROOT_PASSWORD\" || exit 1"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 20s + +volumes: + curbside_postgres_data: + curbside_mysql_data: diff --git a/tutorials/curbside-pickup/database/mysql-init.sql b/tutorials/curbside-pickup/database/mysql-init.sql new file mode 100644 index 0000000..4678d46 --- /dev/null +++ b/tutorials/curbside-pickup/database/mysql-init.sql @@ -0,0 +1,56 @@ +-- Copyright 2026 The Drasi Authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Curbside Pickup Tutorial - Physical Operations (MySQL) +-- +-- Tracks the pickup vehicles. A driver updates their `location` to 'Curbside' +-- when they arrive at the pickup zone. The Drasi MySQL source captures changes +-- by streaming the binary log (binlog), which the container enables with +-- ROW-based logging, full row images/metadata and GTID mode (see +-- docker-compose.yml). The `drasi_user` (created from MYSQL_USER) is granted +-- replication privileges below so it can read the binlog. +-- +-- The table is `vehicles`; the Drasi source is configured with +-- `tables: [vehicles]`, which yields the node label `vehicles`, matching +-- (v:vehicles) in the Cypher continuous queries. + +USE PhysicalOperations; + +-- vehicles table: one row per pickup vehicle. `plate` is the natural key and the +-- join key to the orders table in PostgreSQL. +DROP TABLE IF EXISTS vehicles; + +CREATE TABLE vehicles ( + plate VARCHAR(10) NOT NULL PRIMARY KEY, + driver_name VARCHAR(50) NOT NULL, + customer_name VARCHAR(50) NOT NULL, + make VARCHAR(50) NOT NULL, + model VARCHAR(50) NOT NULL, + color VARCHAR(30) NOT NULL, + location VARCHAR(20) NOT NULL DEFAULT 'Parking' +); + +-- Seed three vehicles, all parked. The plates match the seeded orders in the +-- PostgreSQL database so the PICKUP_BY join lines up. +INSERT INTO vehicles (plate, driver_name, customer_name, make, model, color, location) VALUES + ('A1234', 'Elijah Brooks', 'Sophia Carter', 'Toyota', 'Camry', 'Blue', 'Parking'), + ('B5678', 'Isabella Reed', 'Mason Rivera', 'Ford', 'F-150', 'Red', 'Parking'), + ('C9876', 'Liam Bennett', 'Ava Mitchell', 'Honda', 'Civic', 'Black', 'Parking'); + +-- Grant the Drasi user the privileges it needs to stream the binlog. MYSQL_USER +-- (drasi_user) already exists with full access to the PhysicalOperations +-- database; it additionally needs the server-wide replication privileges to open +-- a binlog stream. +GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'drasi_user'@'%'; +FLUSH PRIVILEGES; diff --git a/tutorials/curbside-pickup/database/postgres-init.sql b/tutorials/curbside-pickup/database/postgres-init.sql new file mode 100644 index 0000000..b4a9528 --- /dev/null +++ b/tutorials/curbside-pickup/database/postgres-init.sql @@ -0,0 +1,97 @@ +-- Copyright 2026 The Drasi Authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Curbside Pickup Tutorial - Retail Operations (PostgreSQL) +-- +-- Holds the customer orders. The retail team marks an order 'ready' when it is +-- prepared. The `orders` table is lower-case and unquoted so the node label +-- Drasi sees matches the Cypher continuous queries, which use (o:orders) and +-- o.status, o.plate, o.driver_name, o.customer_name without any change. + +-- Suppress noisy output during setup. +\set QUIET on +SET client_min_messages = ERROR; + +-- Create a user with replication privileges for CDC. +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'drasi_user') THEN + CREATE USER drasi_user WITH REPLICATION LOGIN PASSWORD 'drasi_password'; + END IF; +END +$$; + +-- Grant permissions on the database. +GRANT CREATE ON DATABASE "RetailOperations" TO drasi_user; +GRANT ALL PRIVILEGES ON DATABASE "RetailOperations" TO drasi_user; + +-- Drop existing table if it exists. +DROP TABLE IF EXISTS orders CASCADE; + +-- orders table: one row per customer order awaiting curbside pickup. +CREATE TABLE orders ( + id SERIAL PRIMARY KEY, + customer_name VARCHAR(255) NOT NULL, + driver_name VARCHAR(255) NOT NULL, + plate VARCHAR(50) NOT NULL, + status VARCHAR(50) NOT NULL DEFAULT 'preparing' +); + +-- Enforce the order status enum. +ALTER TABLE orders ADD CONSTRAINT chk_status CHECK (status IN ('preparing', 'ready')); + +-- Set REPLICA IDENTITY to FULL so change events include every column. +ALTER TABLE orders REPLICA IDENTITY FULL; + +-- Ensure drasi_user owns the table. +ALTER TABLE orders OWNER TO drasi_user; + +-- Grant permissions to drasi_user. +GRANT USAGE ON SCHEMA public TO drasi_user; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO drasi_user; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO drasi_user; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO drasi_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO drasi_user; + +-- Create the publication for logical replication. The replication slot itself is +-- NOT created here: the Drasi PostgreSQL source creates it on startup and takes a +-- consistent snapshot, so the rows seeded below are loaded once via bootstrap and +-- are not also replayed as change events (which would double-count them). +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'drasi_curbside_pub') THEN + CREATE PUBLICATION drasi_curbside_pub FOR TABLE orders; + END IF; +END +$$; + +-- Seed three orders, all 'preparing'. The plates match the seeded vehicles in the +-- MySQL database so the PICKUP_BY join lines up. +INSERT INTO orders (customer_name, driver_name, plate, status) +SELECT * FROM (VALUES + ('Sophia Carter', 'Elijah Brooks', 'A1234', 'preparing'), + ('Mason Rivera', 'Isabella Reed', 'B5678', 'preparing'), + ('Ava Mitchell', 'Liam Bennett', 'C9876', 'preparing') +) AS d(customer_name, driver_name, plate, status) +WHERE NOT EXISTS (SELECT 1 FROM orders); + +-- Summary. +SET client_min_messages = NOTICE; +DO $$ +BEGIN + RAISE NOTICE 'Retail Operations database initialized successfully!'; + RAISE NOTICE 'Table: orders'; + RAISE NOTICE 'Publication: drasi_curbside_pub'; +END +$$; diff --git a/tutorials/curbside-pickup/engine.py b/tutorials/curbside-pickup/engine.py new file mode 100644 index 0000000..95a4397 --- /dev/null +++ b/tutorials/curbside-pickup/engine.py @@ -0,0 +1,447 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The embedded Drasi engine that powers the Streamlit UI. + +Everything the demo needs on the Drasi side lives in this one file: the two +databases' connection settings and source configs, the synthetic join, the six +continuous queries, and the ``CurbsideEngine`` that wires them together. + +The engine runs Drasi on its own asyncio event loop in a background daemon +thread (created once, cached by Streamlit), and a single **Python reaction** over +all six queries keeps a thread-safe snapshot the UI renders. Drasi reads changes +from a PostgreSQL ``orders`` table and a MySQL ``vehicles`` table and joins them +by license plate. The UI's controls write order changes to Postgres (via +``psycopg``) and vehicle changes to MySQL (via ``PyMySQL``); Drasi observes both +through CDC. Each write is also appended to an in-memory activity log the UI shows. +""" + +from __future__ import annotations + +import os + +# Quiet the engine's default INFO logging. Must run before ``drasi`` is imported. +os.environ.setdefault("RUST_LOG", "warn") + +import asyncio # noqa: E402 +import threading # noqa: E402 +import time # noqa: E402 +from collections import deque # noqa: E402 +from dataclasses import dataclass, field # noqa: E402 +from typing import Any # noqa: E402 + +import psycopg # noqa: E402 +import pymysql # noqa: E402 + +from drasi import Drasi # noqa: E402 +from drasi.types import QueryResultEvent # noqa: E402 + +# ============================================================================= +# Configuration -- two databases, read from the environment with defaults. +# ============================================================================= + +# --- PostgreSQL (orders) --- +POSTGRES = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5742")), + "database": os.environ.get("POSTGRES_DATABASE", "RetailOperations"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), +} +# The Postgres source creates its own replication slot on startup and reuses the +# publication created by database/postgres-init.sql. +POSTGRES_SOURCE_CONFIG = { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.orders"], + "tableKeys": [{"table": "orders", "keyColumns": ["id"]}], + "slotName": "drasi_curbside_slot", + "publicationName": "drasi_curbside_pub", +} +POSTGRES_BOOTSTRAP_CONFIG = {"kind": "postgres", **POSTGRES} +# psycopg spells the database key ``dbname``; the UI writes order status changes +# through this connection so Drasi observes them via logical replication. +PSYCOPG_CONNECTION = { + "host": POSTGRES["host"], + "port": POSTGRES["port"], + "dbname": POSTGRES["database"], + "user": POSTGRES["user"], + "password": POSTGRES["password"], +} + +# --- MySQL (vehicles) --- +MYSQL = { + "host": os.environ.get("MYSQL_HOST", "localhost"), + "port": int(os.environ.get("MYSQL_PORT", "3309")), + "database": os.environ.get("MYSQL_DATABASE", "PhysicalOperations"), + "user": os.environ.get("MYSQL_USER", "drasi_user"), + "password": os.environ.get("MYSQL_PASSWORD", "drasi_password"), +} +# The MySQL source streams the binlog. TLS is disabled on the tutorial container, +# so the source connects in plaintext. Unlike Postgres, the MySQL bootstrap +# provider takes its own connection settings (and does not accept sslMode). +MYSQL_SOURCE_CONFIG = { + **MYSQL, + "sslMode": "disabled", + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], +} +MYSQL_BOOTSTRAP_CONFIG = { + "kind": "mysql", + **MYSQL, + "tables": ["vehicles"], + "tableKeys": [{"table": "vehicles", "keyColumns": ["plate"]}], +} +# PyMySQL spells the database key ``database``; the UI writes vehicle location +# changes through this connection so Drasi observes them via the binlog. +PYMYSQL_CONNECTION = { + "host": MYSQL["host"], + "port": MYSQL["port"], + "database": MYSQL["database"], + "user": MYSQL["user"], + "password": MYSQL["password"], + "autocommit": True, +} + +ORDER_PREPARING = "preparing" +VEHICLE_PARKING = "Parking" + +# How long (matching the delay query) a curbside vehicle waits on an unready order +# before it is flagged as delayed. Shown in the UI copy. +DELAY_SECONDS = 10 +# How many recent SQL writes to keep in the activity log. +ACTIVITY_LOG_SIZE = 20 + +# ============================================================================= +# The six continuous queries. +# ============================================================================= + +# Source ids (one per database). +RETAIL_OPS = "retail-ops" # PostgreSQL: orders +PHYSICAL_OPS = "physical-ops" # MySQL: vehicles + +# Query ids -- the UI reads each query's result set from the snapshot by these. +ORDERS_PREPARING = "orders-preparing" +ORDERS_READY = "orders-ready" +VEHICLES_PARKING = "vehicles-parking" +VEHICLES_CURBSIDE = "vehicles-curbside" +DELIVERY = "delivery" +DELAY = "delay" + + +@dataclass(frozen=True) +class Query: + """One continuous query: how to register it and how to index its rows.""" + + id: str + key: str # the RETURN column that identifies each row (its primary key) + sources: list[str] # the source ids this query reads from + cypher: str + joins: list[dict[str, Any]] = field(default_factory=list) + + +# The four single-source list queries split orders and vehicles by state, and the +# two join queries (delivery, delay) relate them across the two databases. +QUERIES = [ + # Orders still being prepared (status != 'ready'). + Query( + id=ORDERS_PREPARING, + key="id", + sources=[RETAIL_OPS], + cypher=""" + MATCH (o:orders) + WHERE o.status <> 'ready' + RETURN + o.id AS id, + o.id AS orderId, + o.customer_name AS customerName, + o.driver_name AS driverName, + o.plate AS plate, + o.status AS status + """, + ), + # Orders that are ready for pickup (status = 'ready'). + Query( + id=ORDERS_READY, + key="id", + sources=[RETAIL_OPS], + cypher=""" + MATCH (o:orders) + WHERE o.status = 'ready' + RETURN + o.id AS id, + o.id AS orderId, + o.customer_name AS customerName, + o.driver_name AS driverName, + o.plate AS plate, + o.status AS status + """, + ), + # Vehicles still in the parking lot (location = 'Parking'). + Query( + id=VEHICLES_PARKING, + key="id", + sources=[PHYSICAL_OPS], + cypher=""" + MATCH (v:vehicles) + WHERE v.location = 'Parking' + RETURN + v.plate AS id, + v.plate AS plate, + v.make AS make, + v.model AS model, + v.color AS color, + v.location AS location + """, + ), + # Vehicles waiting at the curb (location = 'Curbside'). + Query( + id=VEHICLES_CURBSIDE, + key="id", + sources=[PHYSICAL_OPS], + cypher=""" + MATCH (v:vehicles) + WHERE v.location = 'Curbside' + RETURN + v.plate AS id, + v.plate AS plate, + v.make AS make, + v.model AS model, + v.color AS color, + v.location AS location + """, + ), + # Matched orders: ready AND the driver's vehicle is at the curbside. There is + # no foreign key between the two databases, so the query declares the + # relationship as a synthetic join, matching a vehicle to an order by equal + # plate. drasi.listMax picks the later of the two change times. + Query( + id=DELIVERY, + key="id", + sources=[RETAIL_OPS, PHYSICAL_OPS], + joins=[ + { + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], + } + ], + cypher=""" + MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) + WHERE o.status = 'ready' AND v.location = 'Curbside' + RETURN + o.id AS id, + o.id AS orderId, + o.customer_name AS customerName, + o.driver_name AS driverName, + o.plate AS vehicleId, + v.make AS vehicleMake, + v.model AS vehicleModel, + v.color AS vehicleColor, + v.location AS vehicleLocation, + drasi.listMax([drasi.changeDateTime(o), drasi.changeDateTime(v)]) AS readyTimestamp + """, + ), + # Delayed orders: a driver has been at the curbside for over 10 seconds while + # the order is still not ready. drasi.trueFor schedules a future re-evaluation + # and fires the instant the condition has held for the given duration. + Query( + id=DELAY, + key="orderId", + sources=[RETAIL_OPS, PHYSICAL_OPS], + joins=[ + { + "id": "PICKUP_BY", + "keys": [ + {"label": "vehicles", "property": "plate"}, + {"label": "orders", "property": "plate"}, + ], + } + ], + cypher=""" + MATCH (o:orders)-[:PICKUP_BY]->(v:vehicles) + WHERE o.status <> 'ready' + AND drasi.trueFor(v.location = 'Curbside', duration({ seconds: 10 })) + RETURN + o.id AS orderId, + o.customer_name AS customerName, + o.driver_name AS driverName, + o.plate AS plate, + drasi.changeDateTime(v) AS waitingSinceTimestamp + """, + ), +] + +# ============================================================================= +# The engine. +# ============================================================================= + + +class CurbsideEngine: + """Runs Drasi over two databases and exposes a thread-safe snapshot.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + # Per query: a dict keyed by that query's primary-key field. The reaction + # applies diffs into it; the UI reads a snapshot of the values. + self._results: dict[str, dict[Any, dict[str, Any]]] = {} + # Which RETURN column identifies each query's rows, looked up by query id. + self._key_by_id = {query.id: query.key for query in QUERIES} + self._version = 0 + # Recent SQL writes, newest last: {"db": ..., "sql": ..., "at": ...}. + self._activity: deque[dict[str, Any]] = deque(maxlen=ACTIVITY_LOG_SIZE) + + self._loop: asyncio.AbstractEventLoop | None = None + self._drasi: Drasi | None = None + + self._ready = threading.Event() + self._error: BaseException | None = None + + self._thread = threading.Thread(target=self._run, name="drasi-curbside-engine", daemon=True) + self._thread.start() + + # -- lifecycle (background thread) --------------------------------------- + + def _run(self) -> None: + loop = asyncio.new_event_loop() + self._loop = loop + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(self._start()) + except BaseException as exc: # noqa: BLE001 - surfaced to the UI thread + self._error = exc + self._ready.set() + return + self._ready.set() + loop.run_forever() + + async def _start(self) -> None: + self._drasi = await Drasi.create("curbside-pickup") + + for plugin in ( + "source/postgres", + "bootstrap/postgres", + "source/mysql", + "bootstrap/mysql", + ): + await self._drasi.install_plugin(plugin) + await self._drasi.start() + + # Two sources: PostgreSQL orders and MySQL vehicles. + await self._drasi.add_source( + "postgres", RETAIL_OPS, POSTGRES_SOURCE_CONFIG, bootstrap=POSTGRES_BOOTSTRAP_CONFIG + ) + await self._drasi.add_source( + "mysql", PHYSICAL_OPS, MYSQL_SOURCE_CONFIG, bootstrap=MYSQL_BOOTSTRAP_CONFIG + ) + + for query in QUERIES: + await self._drasi.add_query( + query.id, query.cypher, query.sources, joins=query.joins or None + ) + for query in QUERIES: + await self._drasi.wait_for_query(query.id, timeout=60) + + # Prime the snapshot from the bootstrapped result sets, then let the + # reaction keep it current. + for query in QUERIES: + rows = await self._drasi.get_query_results(query.id) + with self._lock: + self._results[query.id] = {row[query.key]: row for row in rows} + + await self._drasi.add_python_reaction( + "ui", [query.id for query in QUERIES], self._on_results + ) + + def _on_results(self, event: QueryResultEvent) -> None: + """Apply a batch of diffs to the snapshot (called synchronously).""" + query_id = event["query_id"] + key_field = self._key_by_id[query_id] + with self._lock: + store = self._results.setdefault(query_id, {}) + for diff in event["results"]: + match diff: + case {"type": "DELETE", "data": dict() as row}: + store.pop(row[key_field], None) + case {"after": dict() as row} | {"data": dict() as row}: + # ADD / UPDATE / aggregation: `after` is the new state + # (aggregation diffs carry a null `data`), `data` covers ADD. + store[row[key_field]] = row + self._version += 1 + + # -- readiness ----------------------------------------------------------- + + def wait_ready(self, timeout: float = 240.0) -> None: + """Block until the engine is running, raising if it failed to start.""" + if not self._ready.wait(timeout): + raise TimeoutError("the Drasi engine did not start in time") + if self._error is not None: + raise RuntimeError(f"the Drasi engine failed to start: {self._error}") + + # -- reads (any thread) -------------------------------------------------- + + def snapshot(self) -> dict[str, Any]: + """Return the current result set of every query, plus the activity log.""" + with self._lock: + results = {qid: list(store.values()) for qid, store in self._results.items()} + activity = list(self._activity) + return { + "version": self._version, + "results": results, + "activity": activity, + "error": str(self._error) if self._error else None, + } + + # -- writes (any thread -> a database, observed via CDC) ----------------- + + def set_order_status(self, order_id: int, status: str) -> None: + """Update one order's status in PostgreSQL.""" + with psycopg.connect(**PSYCOPG_CONNECTION) as conn: + conn.execute("UPDATE orders SET status = %s WHERE id = %s", (status, order_id)) + self._log("PostgreSQL", f"UPDATE orders SET status='{status}' WHERE id={order_id};") + + def set_vehicle_location(self, plate: str, location: str) -> None: + """Update one vehicle's location in MySQL.""" + conn = pymysql.connect(**PYMYSQL_CONNECTION) + try: + with conn.cursor() as cur: + cur.execute("UPDATE vehicles SET location = %s WHERE plate = %s", (location, plate)) + finally: + conn.close() + self._log("MySQL", f"UPDATE vehicles SET location='{location}' WHERE plate='{plate}';") + + def reset(self) -> None: + """Return everything to the start: all orders preparing, all parked.""" + with psycopg.connect(**PSYCOPG_CONNECTION) as conn: + conn.execute( + "UPDATE orders SET status = %s WHERE status <> %s", + (ORDER_PREPARING, ORDER_PREPARING), + ) + conn2 = pymysql.connect(**PYMYSQL_CONNECTION) + try: + with conn2.cursor() as cur: + cur.execute( + "UPDATE vehicles SET location = %s WHERE location <> %s", + (VEHICLE_PARKING, VEHICLE_PARKING), + ) + finally: + conn2.close() + self._log("PostgreSQL", f"UPDATE orders SET status='{ORDER_PREPARING}';") + self._log("MySQL", f"UPDATE vehicles SET location='{VEHICLE_PARKING}';") + + def _log(self, db: str, sql: str) -> None: + with self._lock: + self._activity.append({"db": db, "sql": sql, "at": time.strftime("%H:%M:%S")}) diff --git a/tutorials/curbside-pickup/images/curbside-ui.png b/tutorials/curbside-pickup/images/curbside-ui.png new file mode 100644 index 0000000..d306296 Binary files /dev/null and b/tutorials/curbside-pickup/images/curbside-ui.png differ diff --git a/tutorials/curbside-pickup/requirements.txt b/tutorials/curbside-pickup/requirements.txt new file mode 100644 index 0000000..6a60bca --- /dev/null +++ b/tutorials/curbside-pickup/requirements.txt @@ -0,0 +1,12 @@ +# Curbside Pickup tutorial dependencies. +# +# pip install -r requirements.txt +# +# drasi-lib is the Python binding for Drasi's embeddable engine. It downloads the +# Postgres and MySQL source/bootstrap plugins from ghcr.io at runtime, so the +# first run needs network access. +drasi-lib>=0.2,<1 +streamlit>=1.32 +streamlit-autorefresh>=1.0 +psycopg[binary]>=3.1 +PyMySQL>=1.1 diff --git a/tutorials/curbside-pickup/scripts/cleanup.ps1 b/tutorials/curbside-pickup/scripts/cleanup.ps1 new file mode 100644 index 0000000..8c42f2e --- /dev/null +++ b/tutorials/curbside-pickup/scripts/cleanup.ps1 @@ -0,0 +1,48 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup Script (Windows) +# Stops the PostgreSQL and MySQL containers. The app runs in the foreground, so +# stop it first with Ctrl+C. Pass -RemoveVolumes to also delete the data volumes. + +param( + [switch]$RemoveVolumes +) + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$DatabaseDir = Join-Path $ScriptDir "..\database" + +Write-Host "=== Curbside Pickup - Cleanup ===" +Write-Host "" + +Push-Location $DatabaseDir +try { + if ($RemoveVolumes) { + Write-Host "Removing containers and volumes..." + docker compose down -v 2>&1 | Write-Host + } else { + Write-Host "Removing containers (keeping volumes)..." + docker compose down 2>&1 | Write-Host + } +} +finally { + Pop-Location +} + +Write-Host "" +Write-Host "=== Cleanup complete! ===" +Write-Host "" +Write-Host "Options:" +Write-Host " scripts/cleanup.ps1 # Stop containers, keep data volumes" +Write-Host " scripts/cleanup.ps1 -RemoveVolumes # Stop containers and remove data volumes" diff --git a/tutorials/curbside-pickup/scripts/cleanup.sh b/tutorials/curbside-pickup/scripts/cleanup.sh new file mode 100755 index 0000000..2c1e2e0 --- /dev/null +++ b/tutorials/curbside-pickup/scripts/cleanup.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup Script +# Removes the PostgreSQL and MySQL containers. The app runs in the foreground, so +# stop it first with Ctrl+C. Pass --volumes to also delete the database volumes. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATABASE_DIR="$SCRIPT_DIR/../database" +REMOVE_VOLUMES="${1:-}" + +echo "=== Curbside Pickup - Cleanup ===" +echo + +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null 2>&1; then + COMPOSE_CMD="docker compose" +else + echo "Warning: docker-compose not found, skipping container cleanup" + exit 0 +fi + +echo "Stopping containers..." +cd "$DATABASE_DIR" + +if [ "$REMOVE_VOLUMES" = "--volumes" ] || [ "$REMOVE_VOLUMES" = "-v" ]; then + echo "Removing containers and volumes..." + $COMPOSE_CMD down -v +else + echo "Removing containers (keeping volumes)..." + $COMPOSE_CMD down +fi + +echo +echo "=== Cleanup complete! ===" +echo +echo "Options:" +echo " $0 # Stop containers, keep data volumes" +echo " $0 --volumes # Stop containers and remove data volumes" diff --git a/tutorials/curbside-pickup/scripts/setup-database.ps1 b/tutorials/curbside-pickup/scripts/setup-database.ps1 new file mode 100644 index 0000000..4906460 --- /dev/null +++ b/tutorials/curbside-pickup/scripts/setup-database.ps1 @@ -0,0 +1,94 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup Database Script (Windows) +# Starts PostgreSQL (orders) and MySQL (vehicles) configured for CDC and seeds +# both databases. + +# We check $LASTEXITCODE explicitly rather than setting a terminating error +# preference, because the docker CLI writes progress to stderr. + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$DatabaseDir = Join-Path $ScriptDir "..\database" +$MysqlRootPassword = if ($env:MYSQL_ROOT_PASSWORD) { $env:MYSQL_ROOT_PASSWORD } else { "root_admin" } + +Write-Host "=== Curbside Pickup - Database Setup ===" +Write-Host "" + +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + Write-Host "Error: Docker is not installed or not in PATH" + exit 1 +} + +docker info 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Docker daemon is not running" + exit 1 +} + +Push-Location $DatabaseDir +try { + Write-Host "Stopping any existing containers..." + docker compose down -v 2>&1 | Out-Null + + Write-Host "Starting PostgreSQL and MySQL..." + docker compose up -d 2>&1 | Write-Host + if ($LASTEXITCODE -ne 0) { + Write-Host "Error: failed to start containers" + exit 1 + } + + Write-Host "Waiting for PostgreSQL to be ready..." + $pgReady = $false + for ($i = 1; $i -le 30; $i++) { + docker exec curbside-pickup-postgres pg_isready -h localhost -U postgres -d RetailOperations 2>&1 | Out-Null + if ($LASTEXITCODE -eq 0) { Write-Host "PostgreSQL is ready!"; $pgReady = $true; break } + Write-Host " Waiting... ($i/30)"; Start-Sleep -Seconds 2 + } + if (-not $pgReady) { + Write-Host "Error: PostgreSQL did not become ready within the timeout." + Write-Host "Check logs with: docker logs curbside-pickup-postgres" + exit 1 + } + + Write-Host "Waiting for MySQL to be ready..." + $mysqlReady = $false + for ($i = 1; $i -le 45; $i++) { + docker exec curbside-pickup-mysql mysqladmin ping -h localhost -uroot -p"$MysqlRootPassword" 2>&1 | Out-Null + if ($LASTEXITCODE -eq 0) { Write-Host "MySQL is ready!"; $mysqlReady = $true; break } + Write-Host " Waiting... ($i/45)"; Start-Sleep -Seconds 2 + } + if (-not $mysqlReady) { + Write-Host "Error: MySQL did not become ready within the timeout." + Write-Host "Check logs with: docker logs curbside-pickup-mysql" + exit 1 + } + + Write-Host "" + Write-Host "Seeding PostgreSQL (orders)..." + Get-Content (Join-Path $DatabaseDir "postgres-init.sql") | ` + docker exec -i curbside-pickup-postgres psql -v ON_ERROR_STOP=1 -U postgres -d RetailOperations + if ($LASTEXITCODE -ne 0) { Write-Host "Error: failed to seed PostgreSQL"; exit 1 } + + Write-Host "Seeding MySQL (vehicles)..." + Get-Content (Join-Path $DatabaseDir "mysql-init.sql") | ` + docker exec -i curbside-pickup-mysql mysql -uroot -p"$MysqlRootPassword" + if ($LASTEXITCODE -ne 0) { Write-Host "Error: failed to seed MySQL"; exit 1 } + + Write-Host "" + Write-Host "=== Database setup complete! ===" +} +finally { + Pop-Location +} diff --git a/tutorials/curbside-pickup/scripts/setup-database.sh b/tutorials/curbside-pickup/scripts/setup-database.sh new file mode 100755 index 0000000..7c2b9af --- /dev/null +++ b/tutorials/curbside-pickup/scripts/setup-database.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup Database Script +# Starts PostgreSQL (orders, logical replication) and MySQL (vehicles, ROW-based +# binlog + GTID) and seeds both, so Drasi can stream their changes via CDC. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATABASE_DIR="$SCRIPT_DIR/../database" +MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-root_admin}" + +echo "=== Curbside Pickup - Database Setup ===" +echo + +if ! command -v docker &> /dev/null; then + echo "Error: Docker is not installed or not in PATH" + echo "Please install Docker: https://docs.docker.com/get-docker/" + exit 1 +fi + +if ! docker info &> /dev/null; then + echo "Error: Docker daemon is not running" + echo "Please start Docker and try again" + exit 1 +fi + +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null 2>&1; then + COMPOSE_CMD="docker compose" +else + echo "Error: docker-compose is not installed" + echo "Please install Docker Compose: https://docs.docker.com/compose/install/" + exit 1 +fi + +echo "Using: $COMPOSE_CMD" +echo + +echo "Stopping any existing containers..." +cd "$DATABASE_DIR" +$COMPOSE_CMD down -v 2>/dev/null || true + +echo "Starting PostgreSQL and MySQL..." +$COMPOSE_CMD up -d + +echo "Waiting for PostgreSQL to be ready..." +POSTGRES_READY=false +for i in $(seq 1 30); do + if docker exec curbside-pickup-postgres pg_isready -h localhost -U postgres -d RetailOperations &> /dev/null; then + echo "PostgreSQL is ready!" + POSTGRES_READY=true + break + fi + echo " Waiting... ($i/30)" + sleep 2 +done + +if [ "$POSTGRES_READY" != "true" ]; then + echo "Error: PostgreSQL did not become ready within the timeout." + echo "Check logs with: docker logs curbside-pickup-postgres" + exit 1 +fi + +echo "Waiting for MySQL to be ready..." +MYSQL_READY=false +for i in $(seq 1 45); do + if docker exec curbside-pickup-mysql mysqladmin ping -h localhost -uroot -p"$MYSQL_ROOT_PASSWORD" &> /dev/null; then + echo "MySQL is ready!" + MYSQL_READY=true + break + fi + echo " Waiting... ($i/45)" + sleep 2 +done + +if [ "$MYSQL_READY" != "true" ]; then + echo "Error: MySQL did not become ready within the timeout." + echo "Check logs with: docker logs curbside-pickup-mysql" + exit 1 +fi + +echo +echo "Seeding PostgreSQL (orders)..." +docker exec -i curbside-pickup-postgres \ + psql -v ON_ERROR_STOP=1 -U postgres -d RetailOperations < "$DATABASE_DIR/postgres-init.sql" + +echo "Seeding MySQL (vehicles)..." +docker exec -i curbside-pickup-mysql \ + mysql -uroot -p"$MYSQL_ROOT_PASSWORD" < "$DATABASE_DIR/mysql-init.sql" + +echo +echo "Seeded orders (PostgreSQL):" +docker exec curbside-pickup-postgres psql -U drasi_user -d RetailOperations -c \ + "SELECT id, customer_name, plate, status FROM orders ORDER BY id;" + +echo "Seeded vehicles (MySQL):" +docker exec curbside-pickup-mysql \ + mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -D PhysicalOperations -e \ + "SELECT plate, make, model, color, location FROM vehicles ORDER BY plate;" 2>/dev/null + +echo +echo "=== Database setup complete! ===" +echo " PostgreSQL: localhost:${POSTGRES_HOST_PORT:-5742} / RetailOperations" +echo " MySQL: localhost:${MYSQL_HOST_PORT:-3309} / PhysicalOperations" diff --git a/tutorials/curbside-pickup/scripts/start-demo.ps1 b/tutorials/curbside-pickup/scripts/start-demo.ps1 new file mode 100644 index 0000000..febbaae --- /dev/null +++ b/tutorials/curbside-pickup/scripts/start-demo.ps1 @@ -0,0 +1,41 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start Demo Script (Windows) +# Starts (and seeds) both databases, then launches the Streamlit app in the +# foreground. Open the URL it prints (http://localhost:8501 by default) and drive +# the demo from the panels. Press Ctrl+C to stop, then run scripts/cleanup.ps1. + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$TutorialDir = Join-Path $ScriptDir ".." + +& (Join-Path $ScriptDir "setup-database.ps1") +if ($LASTEXITCODE -ne 0) { + Write-Host "Database setup failed; not starting the app." + exit 1 +} + +Write-Host "" +Write-Host "Starting the Streamlit app (embeds the Drasi engine)..." +Write-Host "On first run it downloads the Drasi plugins from ghcr.io - give it a moment." +Write-Host "" + +$port = if ($env:STREAMLIT_PORT) { $env:STREAMLIT_PORT } else { "8501" } +Push-Location $TutorialDir +try { + streamlit run app.py --server.port $port --server.address=0.0.0.0 --server.headless=true +} +finally { + Pop-Location +} diff --git a/tutorials/curbside-pickup/scripts/start-demo.sh b/tutorials/curbside-pickup/scripts/start-demo.sh new file mode 100755 index 0000000..4a579f1 --- /dev/null +++ b/tutorials/curbside-pickup/scripts/start-demo.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start Demo Script +# Starts (and seeds) both databases, then launches the Streamlit app in the +# foreground. The app embeds the Drasi engine, so this is the only process you +# need. Open the URL it prints (http://localhost:8501 by default) and drive the +# demo from the panels. Press Ctrl+C to stop, then run ./scripts/cleanup.sh. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TUTORIAL_DIR="$SCRIPT_DIR/.." + +bash "$SCRIPT_DIR/setup-database.sh" + +echo +echo "Starting the Streamlit app (embeds the Drasi engine)..." +echo "On first run it downloads the Drasi plugins from ghcr.io — give it a moment." +echo + +cd "$TUTORIAL_DIR" +exec streamlit run app.py \ + --server.port "${STREAMLIT_PORT:-8501}" \ + --server.address=0.0.0.0 \ + --server.headless=true diff --git a/tutorials/getting-started/.env.example b/tutorials/getting-started/.env.example new file mode 100644 index 0000000..db51a0d --- /dev/null +++ b/tutorials/getting-started/.env.example @@ -0,0 +1,16 @@ +# Example environment variables for the getting-started tutorial. +# +# Copy this file to .env and adjust as needed: +# cp .env.example .env +# +# The scripts and the app read these with the same defaults shown here. +# +# IMPORTANT: Never commit .env with real credentials! + +# PostgreSQL connection settings. +POSTGRES_HOST=localhost +POSTGRES_PORT=5752 +POSTGRES_HOST_PORT=5752 +POSTGRES_DATABASE=getting_started +POSTGRES_USER=drasi_user +POSTGRES_PASSWORD=drasi_password diff --git a/tutorials/getting-started/.gitignore b/tutorials/getting-started/.gitignore new file mode 100644 index 0000000..1e58996 --- /dev/null +++ b/tutorials/getting-started/.gitignore @@ -0,0 +1,11 @@ +# Local environment overrides +.env + +# Python +__pycache__/ +*.pyc +.venv/ + +# Drasi engine state / logs +data/ +*.log diff --git a/tutorials/getting-started/README.md b/tutorials/getting-started/README.md new file mode 100644 index 0000000..23a5376 --- /dev/null +++ b/tutorials/getting-started/README.md @@ -0,0 +1,378 @@ +<!-- DO NOT EDIT. Generated from _index.md by scripts/render-tutorials.py. Edit _index.md and run `python3 scripts/render-tutorials.py`. --> + +Imagine you want to react the instant data changes — a new row in a database, a value crossing a threshold, or something that *should* have changed but didn't. Drasi lets you express these as **continuous queries** that stay constantly up to date, with no polling. + +This is the getting-started tutorial for **`drasi-lib`**, the Python binding that embeds Drasi's engine directly in your app. You'll connect Drasi to a live PostgreSQL table of messages — imagine a simple live message feed — and build three continuous queries that **filter** changes, **aggregate** them, and detect the **absence** of change. A single **Python reaction** prints each query's changes to the console, so there's no UI to build: you run one small console app, drive changes with plain SQL, and watch the queries react in your terminal. + +**Sources** → **Continuous Queries** → **Reactions** + +- **Sources** — Connect to your data sources +- **Continuous Queries** — Define what changes matter +- **Reactions** — Take action automatically + +| Step | What You'll Do | +| ---- | ------------- | +| **[Step 1: Set Up Your Environment](#step-1-of-4-set-up-your-environment)** | Open the dev container (or install Python + Docker locally) | +| **[Step 2: Run the Demo](#step-2-of-4-run-the-demo)** | One command starts PostgreSQL and the console app | +| **[Step 3: Read the Console](#step-3-of-4-read-the-console)** | See the three queries' initial results | +| **[Step 4: Drive Change](#step-4-of-4-drive-change)** | Add and delete messages with SQL, and watch the queries react | +| **[How It Works](#how-it-works)** | Understand the source, the three queries, and the Python reaction | + +> **Before you begin** +> +> - **Two terminals:** **Terminal 1** runs the console app (it stays in the foreground and prints changes). Use **Terminal 2** to change messages with SQL. +> - **Working directory:** run every command from the tutorial directory (`tutorials/getting-started/`). The dev container opens there automatically; if you're running locally, `cd tutorials/getting-started` first. +> - **Command tabs:** commands are shown in tabs (*bash / zsh* and *PowerShell*). Use the one for your shell. +> - **Ports:** PostgreSQL is published on `5752`. There is no web UI. + +## Step 1 of 4: Set Up Your Environment +This tutorial needs **Python 3.10+** and **Docker** (for PostgreSQL). The easiest way to get everything is the **dev container**. + +### Option A: Dev Container or GitHub Codespaces (recommended) + +1. Open the [`drasi-python`](https://github.com/drasi-project/drasi-python) repository in VS Code and run **Reopen in Container** (or create a **Codespace** from the repo's **Code** menu). +2. When prompted for a configuration, choose **Getting Started Tutorial (Python)**. +3. Wait for the container to finish. Its setup script installs the tutorial's Python dependency (`drasi-lib`) and the Docker tooling for PostgreSQL. + +That's it. Skip ahead to [Step 2](#step-2-of-4-run-the-demo). + +### Option B: Run Locally + +You'll need **Python 3.10+**, **Docker**, and **bash** for the helper scripts. From the repository root, move into the tutorial directory and install the dependency: + +**bash / zsh** + +```bash +cd tutorials/getting-started +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +``` + +**PowerShell** + +```powershell +cd tutorials/getting-started +python -m venv .venv; .venv\Scripts\Activate.ps1 +pip install -r requirements.txt +``` + +`drasi-lib` ships as a prebuilt wheel, so no Rust toolchain is needed. It downloads the Postgres plugins it needs from `ghcr.io/drasi-project` the first time the app runs. + +## Step 2 of 4: Run the Demo +In **Terminal 1**, start the demo: + +**bash / zsh** + +```bash +bash scripts/start-demo.sh +``` + +**PowerShell** + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/start-demo.ps1 +``` + +The `start-demo` script starts PostgreSQL (seeding three messages, one of them exactly `Hello World`) and then runs the console app, which embeds the Drasi engine. On first start it downloads the plugins it needs (`source/postgres`, `bootstrap/postgres`) from `ghcr.io/drasi-project`, connects to the database, bootstraps the existing rows, and starts the three queries and the reaction. + +Leave this running. Everything else happens in **Terminal 2**. + +> **Stopping and resetting** +> +> Press **Ctrl+C** in Terminal 1 to stop the app. To remove the database container when you're completely done, run `bash scripts/cleanup.sh` (bash) or `powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1` (PowerShell). Add `--volumes` (bash) or `-RemoveVolumes` (PowerShell) to also delete the data. + +## Step 3 of 4: Read the Console +Once it's ready, the app prints each query's initial results and then waits for changes: + +```text +=== Initial results === + +[hello-world-from] 1 row(s): + MessageFrom='Brian Kernighan' MessageId=2 + +[message-count] 3 row(s): + Frequency=1 Message='Hello World' + Frequency=1 Message='To infinity and beyond!' + Frequency=1 Message='The Analytical Engine weaves algebraic patterns.' + +[inactive-people] 0 row(s): + +=== Watching for changes (Ctrl+C to stop) === +``` + +`hello-world-from` already found the seeded `Hello World`; `message-count` shows each message once; `inactive-people` is empty. Because the seed messages then sit untouched, after about 20 seconds each sender crosses the inactivity threshold and the temporal query fires on its own: + +```text +[13:06:09] [inactive-people] + LastMessageTimestamp='...T20:05:49+00:00' MessageFrom='Buzz Lightyear' +[13:06:09] [inactive-people] + LastMessageTimestamp='...T20:05:49+00:00' MessageFrom='Ada Lovelace' +[13:06:09] [inactive-people] + LastMessageTimestamp='...T20:05:49+00:00' MessageFrom='Brian Kernighan' +``` + +## Step 4 of 4: Drive Change +With Terminal 1 running the app, use **Terminal 2** to change the data and watch it react. The commands below use `docker exec` to run SQL inside the database container, and are identical in bash and PowerShell. + +> **No middle tier — just SQL** +> +> Each command runs a plain SQL `INSERT` or `DELETE` against PostgreSQL — exactly what an existing app would already do. There's no API to call and no event to publish. Drasi observes the row change through logical replication (CDC), re-evaluates the affected queries, and calls the Python reaction, which prints the change. + +### Add a message + +Send a new `Hello World`, from someone new: + +```bash +docker exec getting-started-postgres psql -U drasi_user -d getting_started \ + -c "INSERT INTO message (sender, message) VALUES ('Grace Hopper', 'Hello World') RETURNING messageid;" +``` + +Terminal 1 reacts immediately — the filter picks up the new sender and the aggregate ticks up: + +```text +[13:06:26] [hello-world-from] + MessageFrom='Grace Hopper' MessageId=4 +[13:06:26] [message-count] ~ Frequency=1 Message='Hello World' -> Frequency=2 Message='Hello World' +``` + +Try a message that *isn't* `Hello World` and only `message-count` changes — the filter query ignores it: + +```bash +docker exec getting-started-postgres psql -U drasi_user -d getting_started \ + -c "INSERT INTO message (sender, message) VALUES ('Bob', 'Goodbye World');" +``` + +### Delete a message + +Remove a message by its id (the `INSERT` above prints the new id via `RETURNING`; message `2` is the seeded `Hello World`): + +```bash +docker exec getting-started-postgres psql -U drasi_user -d getting_started \ + -c "DELETE FROM message WHERE messageid=2;" +``` + +`hello-world-from` drops that row and `message-count` decrements — a row leaving a result set is a change Drasi reports too. + +## How It Works +The whole demo is a single file, `tutorials/getting-started/app.py`. Here's what each part does; the [complete file](#putting-it-all-together) is at the end. + +### The Source + +The app installs the Postgres plugins and adds the source, pointing it at the tutorial database: + +```python +await drasi.install_plugin("source/postgres") +await drasi.install_plugin("bootstrap/postgres") +await drasi.start() + +await drasi.add_source( + "postgres", + "messages", + { + **POSTGRES, # host, port, user, password, database + "sslMode": "prefer", + "tables": ["public.message"], + "tableKeys": [{"table": "message", "keyColumns": ["messageid"]}], + "slotName": "drasi_getting_started_slot", + "publicationName": "drasi_getting_started_pub", + }, + bootstrap={"kind": "postgres", **POSTGRES}, +) +``` + +The source uses **logical replication (CDC)** to stream changes from the `message` table; the `bootstrap` provider loads the rows that already exist when the query starts. The table name is lower-case so the node label Drasi sees matches the queries: `(m:message)`. + +### The Three Queries + +They build up in complexity. + +**Filter** — `hello-world-from` returns only messages whose text is exactly `Hello World`, with who sent them: + +```cypher +MATCH (m:message) +WHERE m.message = 'Hello World' +RETURN m.messageid AS MessageId, m.sender AS MessageFrom +``` + +**Aggregation** — `message-count` groups by message text and counts. Like SQL's `GROUP BY`, the non-aggregated `Message` travels alongside `count(m)`, and Drasi maintains the counts **incrementally** — one insert bumps exactly one row: + +```cypher +MATCH (m:message) +RETURN m.message AS Message, count(m) AS Frequency +``` + +**Absence of change** — `inactive-people` finds senders who have gone quiet. It takes each sender's most recent message time and flags them once it is more than 20 seconds ago. `drasi.trueLater` **schedules a future re-evaluation** so a sender appears the instant they cross the threshold, not only when some other change happens: + +```cypher +MATCH (m:message) +WITH m.sender AS MessageFrom, max(drasi.changeDateTime(m)) AS LastMessageTimestamp +WHERE LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }) + OR drasi.trueLater( + LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }), + LastMessageTimestamp + duration({ seconds: 20 }) + ) +RETURN MessageFrom, LastMessageTimestamp +``` + +`drasi.changeDateTime(m)` exposes the wall-clock time of each change and `datetime.realtime()` is the current time, so this needs no extra timestamp bookkeeping in your app. + +### The Python Reaction + +Instead of a dashboard or an SSE stream, a single **Python reaction** subscribed to all three queries prints each change to the console. Structural pattern matching reads the diff's `type` and pulls out the payload it carries in one step: + +```python +def on_change(event): + query_id = event["query_id"] + for diff in event["results"]: + match diff: + case {"type": "ADD", "data": data}: + print(f"[{query_id}] + {data}") + case {"type": "DELETE", "data": data}: + print(f"[{query_id}] - {data}") + case {"type": "UPDATE" | "aggregation", "before": before, "after": after}: + print(f"[{query_id}] ~ {before} -> {after}") + + +await drasi.add_python_reaction( + "console", ["hello-world-from", "message-count", "inactive-people"], on_change +) +``` + +An `ADD` and a `DELETE` carry `data`; an `UPDATE` — and an `aggregation` change, like a `count` ticking up — carries `before` and `after`. After registering the reaction the app just `await asyncio.Event().wait()`s, so it stays in the foreground printing changes until you press Ctrl+C. See [Python reactions](../../guides/python-reactions/) for the full callback contract, or [Streaming results](../../guides/streaming/) to consume the same changes as an async iterator instead. + +## Putting It All Together +The sections above walked through the program one piece at a time. Here is the complete `app.py` that `python app.py` actually runs — the source, the three queries, the reaction, and the loop, in a single file: + +```python +import asyncio +import os +import sys +import time + +os.environ.setdefault("RUST_LOG", "warn") # quiet the engine's INFO logging + +from drasi import Drasi + +# --- The source: the tutorial's PostgreSQL database --- +POSTGRES = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5752")), + "database": os.environ.get("POSTGRES_DATABASE", "getting_started"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), +} +SOURCE_CONFIG = { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.message"], + "tableKeys": [{"table": "message", "keyColumns": ["messageid"]}], + "slotName": "drasi_getting_started_slot", + "publicationName": "drasi_getting_started_pub", +} +BOOTSTRAP_CONFIG = {"kind": "postgres", **POSTGRES} + + +def row(r): + return " ".join(f"{k}={v!r}" for k, v in r.items()) + + +# --- The reaction: print each result-set change --- +def on_change(event): + query_id = event["query_id"] + for diff in event["results"]: + match diff: + case {"type": "ADD", "data": data}: + print(f"[{query_id}] + {row(data)}") + case {"type": "DELETE", "data": data}: + print(f"[{query_id}] - {row(data)}") + case {"type": "UPDATE" | "aggregation", "before": before, "after": after}: + print(f"[{query_id}] ~ {row(before)} -> {row(after)}") + + +async def main(): + sys.stdout.reconfigure(line_buffering=True) + async with await Drasi.create("getting-started") as drasi: + await drasi.install_plugin("source/postgres") + await drasi.install_plugin("bootstrap/postgres") + await drasi.start() + + await drasi.add_source("postgres", "messages", SOURCE_CONFIG, bootstrap=BOOTSTRAP_CONFIG) + + # Add the three continuous queries, in place: a filter, an aggregation, + # and a temporal query that detects the absence of change. + await drasi.add_query( + "hello-world-from", + """ + MATCH (m:message) + WHERE m.message = 'Hello World' + RETURN m.messageid AS MessageId, m.sender AS MessageFrom + """, + ["messages"], + ) + await drasi.add_query( + "message-count", + """ + MATCH (m:message) + RETURN m.message AS Message, count(m) AS Frequency + """, + ["messages"], + ) + await drasi.add_query( + "inactive-people", + """ + MATCH (m:message) + WITH m.sender AS MessageFrom, max(drasi.changeDateTime(m)) AS LastMessageTimestamp + WHERE LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }) + OR drasi.trueLater( + LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }), + LastMessageTimestamp + duration({ seconds: 20 }) + ) + RETURN MessageFrom, LastMessageTimestamp + """, + ["messages"], + ) + + query_ids = ["hello-world-from", "message-count", "inactive-people"] + for query_id in query_ids: + await drasi.wait_for_query(query_id, timeout=60) + + print("=== Initial results ===") + for query_id in query_ids: + for r in await drasi.get_query_results(query_id): + print(f"[{query_id}] {row(r)}") + + await drasi.add_python_reaction("console", query_ids, on_change) + print("=== Watching for changes (Ctrl+C to stop) ===") + await asyncio.Event().wait() + + +asyncio.run(main()) +``` + +That's the whole thing: a live view over a database in one file, with no polling and no UI. + +## Clean Up +When you're finished, stop the app with **Ctrl+C** in Terminal 1, then remove the database container: + +**bash / zsh** + +```bash +# Stop the container, keep data +bash scripts/cleanup.sh + +# Stop the container and delete the data volume +bash scripts/cleanup.sh --volumes +``` + +**PowerShell** + +```powershell +# Stop the container, keep data +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 + +# Stop the container and delete the data volume +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 -RemoveVolumes +``` + +## What You Learned +- **Sources** connect Drasi to live data — here, PostgreSQL via Change Data Capture, in a few lines of Python. +- **Continuous Queries** let you **filter**, **aggregate**, and even detect the **absence** of change (with the temporal `drasi.trueLater`) — all kept current automatically, with no polling. +- A **Python reaction** turned every query change into console output; the same callback could call an API, write a file, or push a notification instead. + +From here, try the [Building Comfort](../building-comfort/) and [Curbside Pickup](../curbside-pickup/) tutorials, which add synthetic joins, cross-database joins, and a live Streamlit UI. diff --git a/tutorials/getting-started/_index.md b/tutorials/getting-started/_index.md new file mode 100644 index 0000000..9b2b52b --- /dev/null +++ b/tutorials/getting-started/_index.md @@ -0,0 +1,412 @@ +--- +type: "docs" +title: "Getting Started" +linkTitle: "Getting Started" +weight: 10 +description: > + Connect Drasi to a live PostgreSQL database and build three continuous queries, step by step, in a small Python console app. +--- + +Imagine you want to react the instant data changes — a new row in a database, a value crossing a threshold, or something that *should* have changed but didn't. Drasi lets you express these as **continuous queries** that stay constantly up to date, with no polling. + +This is the getting-started tutorial for **`drasi-lib`**, the Python binding that embeds Drasi's engine directly in your app. You'll connect Drasi to a live PostgreSQL table of messages — imagine a simple live message feed — and build three continuous queries that **filter** changes, **aggregate** them, and detect the **absence** of change. A single **Python reaction** prints each query's changes to the console, so there's no UI to build: you run one small console app, drive changes with plain SQL, and watch the queries react in your terminal. + +<div class="flow-diagram"> + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-database"></i> + </div> + <div class="flow-step__label">Sources</div> + <div class="flow-step__description">Connect to your data sources</div> + </div> + + <div class="flow-arrow"> + <i class="fas fa-arrow-right"></i> + </div> + + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-filter"></i> + </div> + <div class="flow-step__label">Continuous Queries</div> + <div class="flow-step__description">Define what changes matter</div> + </div> + + <div class="flow-arrow"> + <i class="fas fa-arrow-right"></i> + </div> + + <div class="flow-step"> + <div class="flow-step__icon"> + <i class="fas fa-bolt"></i> + </div> + <div class="flow-step__label">Reactions</div> + <div class="flow-step__description">Take action automatically</div> + </div> +</div> + +| Step | What You'll Do | +| ---- | ------------- | +| **[Step 1: Set Up Your Environment](#setup)** | Open the dev container (or install Python + Docker locally) | +| **[Step 2: Run the Demo](#run)** | One command starts PostgreSQL and the console app | +| **[Step 3: Read the Console](#console)** | See the three queries' initial results | +| **[Step 4: Drive Change](#drive)** | Add and delete messages with SQL, and watch the queries react | +| **[How It Works](#how)** | Understand the source, the three queries, and the Python reaction | + +{{% alert title="Before you begin" color="info" %}} +- **Two terminals:** **Terminal 1** runs the console app (it stays in the foreground and prints changes). Use **Terminal 2** to change messages with SQL. +- **Working directory:** run every command from the tutorial directory (`tutorials/getting-started/`). The dev container opens there automatically; if you're running locally, `cd tutorials/getting-started` first. +- **Command tabs:** commands are shown in tabs (*bash / zsh* and *PowerShell*). Use the one for your shell. +- **Ports:** PostgreSQL is published on `5752`. There is no web UI. +{{% /alert %}} + +## Step 1 of 4: Set Up Your Environment {#setup} + +This tutorial needs **Python 3.10+** and **Docker** (for PostgreSQL). The easiest way to get everything is the **dev container**. + +### Option A: Dev Container or GitHub Codespaces (recommended) + +1. Open the [`drasi-python`](https://github.com/drasi-project/drasi-python) repository in VS Code and run **Reopen in Container** (or create a **Codespace** from the repo's **Code** menu). +2. When prompted for a configuration, choose **Getting Started Tutorial (Python)**. +3. Wait for the container to finish. Its setup script installs the tutorial's Python dependency (`drasi-lib`) and the Docker tooling for PostgreSQL. + +That's it. Skip ahead to [Step 2](#run). + +### Option B: Run Locally + +You'll need **Python 3.10+**, **Docker**, and **bash** for the helper scripts. From the repository root, move into the tutorial directory and install the dependency: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +cd tutorials/getting-started +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +cd tutorials/getting-started +python -m venv .venv; .venv\Scripts\Activate.ps1 +pip install -r requirements.txt +{{< /tab >}} +{{< /tabpane >}} + +`drasi-lib` ships as a prebuilt wheel, so no Rust toolchain is needed. It downloads the Postgres plugins it needs from `ghcr.io/drasi-project` the first time the app runs. + +## Step 2 of 4: Run the Demo {#run} + +In **Terminal 1**, start the demo: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +bash scripts/start-demo.sh +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +powershell -ExecutionPolicy Bypass -File scripts/start-demo.ps1 +{{< /tab >}} +{{< /tabpane >}} + +The `start-demo` script starts PostgreSQL (seeding three messages, one of them exactly `Hello World`) and then runs the console app, which embeds the Drasi engine. On first start it downloads the plugins it needs (`source/postgres`, `bootstrap/postgres`) from `ghcr.io/drasi-project`, connects to the database, bootstraps the existing rows, and starts the three queries and the reaction. + +Leave this running. Everything else happens in **Terminal 2**. + +{{% alert title="Stopping and resetting" color="info" %}} +Press **Ctrl+C** in Terminal 1 to stop the app. To remove the database container when you're completely done, run `bash scripts/cleanup.sh` (bash) or `powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1` (PowerShell). Add `--volumes` (bash) or `-RemoveVolumes` (PowerShell) to also delete the data. +{{% /alert %}} + +## Step 3 of 4: Read the Console {#console} + +Once it's ready, the app prints each query's initial results and then waits for changes: + +```text +=== Initial results === + +[hello-world-from] 1 row(s): + MessageFrom='Brian Kernighan' MessageId=2 + +[message-count] 3 row(s): + Frequency=1 Message='Hello World' + Frequency=1 Message='To infinity and beyond!' + Frequency=1 Message='The Analytical Engine weaves algebraic patterns.' + +[inactive-people] 0 row(s): + +=== Watching for changes (Ctrl+C to stop) === +``` + +`hello-world-from` already found the seeded `Hello World`; `message-count` shows each message once; `inactive-people` is empty. Because the seed messages then sit untouched, after about 20 seconds each sender crosses the inactivity threshold and the temporal query fires on its own: + +```text +[13:06:09] [inactive-people] + LastMessageTimestamp='...T20:05:49+00:00' MessageFrom='Buzz Lightyear' +[13:06:09] [inactive-people] + LastMessageTimestamp='...T20:05:49+00:00' MessageFrom='Ada Lovelace' +[13:06:09] [inactive-people] + LastMessageTimestamp='...T20:05:49+00:00' MessageFrom='Brian Kernighan' +``` + +## Step 4 of 4: Drive Change {#drive} + +With Terminal 1 running the app, use **Terminal 2** to change the data and watch it react. The commands below use `docker exec` to run SQL inside the database container, and are identical in bash and PowerShell. + +{{% alert title="No middle tier — just SQL" color="info" %}} +Each command runs a plain SQL `INSERT` or `DELETE` against PostgreSQL — exactly what an existing app would already do. There's no API to call and no event to publish. Drasi observes the row change through logical replication (CDC), re-evaluates the affected queries, and calls the Python reaction, which prints the change. +{{% /alert %}} + +### Add a message + +Send a new `Hello World`, from someone new: + +```bash +docker exec getting-started-postgres psql -U drasi_user -d getting_started \ + -c "INSERT INTO message (sender, message) VALUES ('Grace Hopper', 'Hello World') RETURNING messageid;" +``` + +Terminal 1 reacts immediately — the filter picks up the new sender and the aggregate ticks up: + +```text +[13:06:26] [hello-world-from] + MessageFrom='Grace Hopper' MessageId=4 +[13:06:26] [message-count] ~ Frequency=1 Message='Hello World' -> Frequency=2 Message='Hello World' +``` + +Try a message that *isn't* `Hello World` and only `message-count` changes — the filter query ignores it: + +```bash +docker exec getting-started-postgres psql -U drasi_user -d getting_started \ + -c "INSERT INTO message (sender, message) VALUES ('Bob', 'Goodbye World');" +``` + +### Delete a message + +Remove a message by its id (the `INSERT` above prints the new id via `RETURNING`; message `2` is the seeded `Hello World`): + +```bash +docker exec getting-started-postgres psql -U drasi_user -d getting_started \ + -c "DELETE FROM message WHERE messageid=2;" +``` + +`hello-world-from` drops that row and `message-count` decrements — a row leaving a result set is a change Drasi reports too. + +## How It Works {#how} + +The whole demo is a single file, `tutorials/getting-started/app.py`. Here's what each part does; the [complete file](#all) is at the end. + +### The Source + +The app installs the Postgres plugins and adds the source, pointing it at the tutorial database: + +```python +await drasi.install_plugin("source/postgres") +await drasi.install_plugin("bootstrap/postgres") +await drasi.start() + +await drasi.add_source( + "postgres", + "messages", + { + **POSTGRES, # host, port, user, password, database + "sslMode": "prefer", + "tables": ["public.message"], + "tableKeys": [{"table": "message", "keyColumns": ["messageid"]}], + "slotName": "drasi_getting_started_slot", + "publicationName": "drasi_getting_started_pub", + }, + bootstrap={"kind": "postgres", **POSTGRES}, +) +``` + +The source uses **logical replication (CDC)** to stream changes from the `message` table; the `bootstrap` provider loads the rows that already exist when the query starts. The table name is lower-case so the node label Drasi sees matches the queries: `(m:message)`. + +### The Three Queries + +They build up in complexity. + +**Filter** — `hello-world-from` returns only messages whose text is exactly `Hello World`, with who sent them: + +```cypher +MATCH (m:message) +WHERE m.message = 'Hello World' +RETURN m.messageid AS MessageId, m.sender AS MessageFrom +``` + +**Aggregation** — `message-count` groups by message text and counts. Like SQL's `GROUP BY`, the non-aggregated `Message` travels alongside `count(m)`, and Drasi maintains the counts **incrementally** — one insert bumps exactly one row: + +```cypher +MATCH (m:message) +RETURN m.message AS Message, count(m) AS Frequency +``` + +**Absence of change** — `inactive-people` finds senders who have gone quiet. It takes each sender's most recent message time and flags them once it is more than 20 seconds ago. `drasi.trueLater` **schedules a future re-evaluation** so a sender appears the instant they cross the threshold, not only when some other change happens: + +```cypher +MATCH (m:message) +WITH m.sender AS MessageFrom, max(drasi.changeDateTime(m)) AS LastMessageTimestamp +WHERE LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }) + OR drasi.trueLater( + LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }), + LastMessageTimestamp + duration({ seconds: 20 }) + ) +RETURN MessageFrom, LastMessageTimestamp +``` + +`drasi.changeDateTime(m)` exposes the wall-clock time of each change and `datetime.realtime()` is the current time, so this needs no extra timestamp bookkeeping in your app. + +### The Python Reaction + +Instead of a dashboard or an SSE stream, a single **Python reaction** subscribed to all three queries prints each change to the console. Structural pattern matching reads the diff's `type` and pulls out the payload it carries in one step: + +```python +def on_change(event): + query_id = event["query_id"] + for diff in event["results"]: + match diff: + case {"type": "ADD", "data": data}: + print(f"[{query_id}] + {data}") + case {"type": "DELETE", "data": data}: + print(f"[{query_id}] - {data}") + case {"type": "UPDATE" | "aggregation", "before": before, "after": after}: + print(f"[{query_id}] ~ {before} -> {after}") + + +await drasi.add_python_reaction( + "console", ["hello-world-from", "message-count", "inactive-people"], on_change +) +``` + +An `ADD` and a `DELETE` carry `data`; an `UPDATE` — and an `aggregation` change, like a `count` ticking up — carries `before` and `after`. After registering the reaction the app just `await asyncio.Event().wait()`s, so it stays in the foreground printing changes until you press Ctrl+C. See [Python reactions](../../guides/python-reactions/) for the full callback contract, or [Streaming results](../../guides/streaming/) to consume the same changes as an async iterator instead. + +## Putting It All Together {#all} + +The sections above walked through the program one piece at a time. Here is the complete `app.py` that `python app.py` actually runs — the source, the three queries, the reaction, and the loop, in a single file: + +```python +import asyncio +import os +import sys +import time + +os.environ.setdefault("RUST_LOG", "warn") # quiet the engine's INFO logging + +from drasi import Drasi + +# --- The source: the tutorial's PostgreSQL database --- +POSTGRES = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5752")), + "database": os.environ.get("POSTGRES_DATABASE", "getting_started"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), +} +SOURCE_CONFIG = { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.message"], + "tableKeys": [{"table": "message", "keyColumns": ["messageid"]}], + "slotName": "drasi_getting_started_slot", + "publicationName": "drasi_getting_started_pub", +} +BOOTSTRAP_CONFIG = {"kind": "postgres", **POSTGRES} + + +def row(r): + return " ".join(f"{k}={v!r}" for k, v in r.items()) + + +# --- The reaction: print each result-set change --- +def on_change(event): + query_id = event["query_id"] + for diff in event["results"]: + match diff: + case {"type": "ADD", "data": data}: + print(f"[{query_id}] + {row(data)}") + case {"type": "DELETE", "data": data}: + print(f"[{query_id}] - {row(data)}") + case {"type": "UPDATE" | "aggregation", "before": before, "after": after}: + print(f"[{query_id}] ~ {row(before)} -> {row(after)}") + + +async def main(): + sys.stdout.reconfigure(line_buffering=True) + async with await Drasi.create("getting-started") as drasi: + await drasi.install_plugin("source/postgres") + await drasi.install_plugin("bootstrap/postgres") + await drasi.start() + + await drasi.add_source("postgres", "messages", SOURCE_CONFIG, bootstrap=BOOTSTRAP_CONFIG) + + # Add the three continuous queries, in place: a filter, an aggregation, + # and a temporal query that detects the absence of change. + await drasi.add_query( + "hello-world-from", + """ + MATCH (m:message) + WHERE m.message = 'Hello World' + RETURN m.messageid AS MessageId, m.sender AS MessageFrom + """, + ["messages"], + ) + await drasi.add_query( + "message-count", + """ + MATCH (m:message) + RETURN m.message AS Message, count(m) AS Frequency + """, + ["messages"], + ) + await drasi.add_query( + "inactive-people", + """ + MATCH (m:message) + WITH m.sender AS MessageFrom, max(drasi.changeDateTime(m)) AS LastMessageTimestamp + WHERE LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }) + OR drasi.trueLater( + LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }), + LastMessageTimestamp + duration({ seconds: 20 }) + ) + RETURN MessageFrom, LastMessageTimestamp + """, + ["messages"], + ) + + query_ids = ["hello-world-from", "message-count", "inactive-people"] + for query_id in query_ids: + await drasi.wait_for_query(query_id, timeout=60) + + print("=== Initial results ===") + for query_id in query_ids: + for r in await drasi.get_query_results(query_id): + print(f"[{query_id}] {row(r)}") + + await drasi.add_python_reaction("console", query_ids, on_change) + print("=== Watching for changes (Ctrl+C to stop) ===") + await asyncio.Event().wait() + + +asyncio.run(main()) +``` + +That's the whole thing: a live view over a database in one file, with no polling and no UI. + +## Clean Up {#cleanup} + +When you're finished, stop the app with **Ctrl+C** in Terminal 1, then remove the database container: + +{{< tabpane persist="header" >}} +{{< tab header="bash / zsh" lang="bash" >}} +# Stop the container, keep data +bash scripts/cleanup.sh + +# Stop the container and delete the data volume +bash scripts/cleanup.sh --volumes +{{< /tab >}} +{{< tab header="PowerShell" lang="powershell" >}} +# Stop the container, keep data +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 + +# Stop the container and delete the data volume +powershell -ExecutionPolicy Bypass -File scripts/cleanup.ps1 -RemoveVolumes +{{< /tab >}} +{{< /tabpane >}} + +## What You Learned {#summary} + +- **Sources** connect Drasi to live data — here, PostgreSQL via Change Data Capture, in a few lines of Python. +- **Continuous Queries** let you **filter**, **aggregate**, and even detect the **absence** of change (with the temporal `drasi.trueLater`) — all kept current automatically, with no polling. +- A **Python reaction** turned every query change into console output; the same callback could call an API, write a file, or push a notification instead. + +From here, try the [Building Comfort](../building-comfort/) and [Curbside Pickup](../curbside-pickup/) tutorials, which add synthetic joins, cross-database joins, and a live Streamlit UI. diff --git a/tutorials/getting-started/app.py b/tutorials/getting-started/app.py new file mode 100644 index 0000000..a99c58f --- /dev/null +++ b/tutorials/getting-started/app.py @@ -0,0 +1,186 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Getting Started -- a console app driven by a Drasi Python reaction. + +Run it with: + + python app.py + +The whole demo is this one file. It connects Drasi's PostgreSQL source to the +tutorial's `message` table, starts three continuous queries, and registers a +single Python reaction that prints each result-set change to the console. Then +it stays running and watches. There is no UI and no prompt: you drive changes by +running SQL directly against the database (with `docker exec ... psql`, shown in +the tutorial) and watch the queries react here in real time. Press Ctrl+C to stop. +""" + +from __future__ import annotations + +import os + +# Quiet the engine's default INFO logging so it doesn't drown out the reaction's +# output. Set RUST_LOG=info (or debug) to see what the engine is doing. Must run +# before ``drasi`` is imported. +os.environ.setdefault("RUST_LOG", "warn") + +import asyncio # noqa: E402 +import sys # noqa: E402 +import time # noqa: E402 +from typing import Any # noqa: E402 + +from drasi import Drasi # noqa: E402 +from drasi.types import QueryResultEvent # noqa: E402 + +# --- The source ------------------------------------------------------------- +# Connection settings for the tutorial's PostgreSQL database, read from the +# environment with the same defaults the database uses. +POSTGRES = { + "host": os.environ.get("POSTGRES_HOST", "localhost"), + "port": int(os.environ.get("POSTGRES_PORT", "5752")), + "database": os.environ.get("POSTGRES_DATABASE", "getting_started"), + "user": os.environ.get("POSTGRES_USER", "drasi_user"), + "password": os.environ.get("POSTGRES_PASSWORD", "drasi_password"), +} + +# The Postgres source streams changes from the `message` table via logical +# replication (CDC). The table is schema-qualified here (public.message) while +# tableKeys uses the bare name; the node label Drasi sees is the bare name, so +# the Cypher matches (m:message). +SOURCE_CONFIG = { + **POSTGRES, + "sslMode": "prefer", + "tables": ["public.message"], + "tableKeys": [{"table": "message", "keyColumns": ["messageid"]}], + "slotName": "drasi_getting_started_slot", + "publicationName": "drasi_getting_started_pub", +} +# The bootstrap provider loads the rows that already exist when a query starts. +BOOTSTRAP_CONFIG = {"kind": "postgres", **POSTGRES} + +# How long a sender can be quiet before inactive-people flags them (matches the +# duration in the inactive-people query below). Shown in the startup banner. +INACTIVITY_SECONDS = 20 + + +def _row(row: dict[str, Any]) -> str: + """Render a result row as a compact, readable `key=value` string.""" + return " ".join(f"{key}={value!r}" for key, value in row.items()) + + +def _stamp() -> str: + return time.strftime("%H:%M:%S") + + +def on_change(event: QueryResultEvent) -> None: + """Print each way a query's result set changed (the console 'reaction').""" + query_id = event["query_id"] + for diff in event["results"]: + match diff: + case {"type": "ADD", "data": data}: + print(f"[{_stamp()}] [{query_id}] + {_row(data)}") + case {"type": "DELETE", "data": data}: + print(f"[{_stamp()}] [{query_id}] - {_row(data)}") + case {"type": "UPDATE" | "aggregation", "before": before, "after": after}: + print(f"[{_stamp()}] [{query_id}] ~ {_row(before)} -> {_row(after)}") + + +async def main() -> None: + # Flush each line as it is printed, so the reaction's output appears live + # even when the app's output is piped to a file rather than a terminal. + sys.stdout.reconfigure(line_buffering=True) + + async with await Drasi.create("getting-started") as drasi: + print("Starting Drasi: installing plugins and connecting to PostgreSQL...") + await drasi.install_plugin("source/postgres") + await drasi.install_plugin("bootstrap/postgres") + await drasi.start() + + await drasi.add_source("postgres", "messages", SOURCE_CONFIG, bootstrap=BOOTSTRAP_CONFIG) + + # Add the three continuous queries, in place. They build up in + # complexity: a filter, an aggregation, and a temporal query that + # detects the *absence* of change. + + # Filter: messages whose text is exactly "Hello World", and who sent them. + await drasi.add_query( + "hello-world-from", + """ + MATCH (m:message) + WHERE m.message = 'Hello World' + RETURN m.messageid AS MessageId, m.sender AS MessageFrom + """, + ["messages"], + ) + + # Aggregation: how many times each distinct message has been sent. + await drasi.add_query( + "message-count", + """ + MATCH (m:message) + RETURN m.message AS Message, count(m) AS Frequency + """, + ["messages"], + ) + + # Absence of change: senders who have not sent a message in the last 20 + # seconds. drasi.trueLater schedules a future re-evaluation so a sender + # appears the instant they cross the threshold, not only when some other + # change happens. + await drasi.add_query( + "inactive-people", + """ + MATCH (m:message) + WITH m.sender AS MessageFrom, max(drasi.changeDateTime(m)) AS LastMessageTimestamp + WHERE LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }) + OR drasi.trueLater( + LastMessageTimestamp <= datetime.realtime() - duration({ seconds: 20 }), + LastMessageTimestamp + duration({ seconds: 20 }) + ) + RETURN MessageFrom, LastMessageTimestamp + """, + ["messages"], + ) + + # Wait for all three to bootstrap their initial result sets. + query_ids = ["hello-world-from", "message-count", "inactive-people"] + for query_id in query_ids: + await drasi.wait_for_query(query_id, timeout=60) + + # Show the state each query bootstrapped from the existing rows. + print("\n=== Initial results ===") + for query_id in query_ids: + rows = await drasi.get_query_results(query_id) + print(f"\n[{query_id}] {len(rows)} row(s):") + for row in rows: + print(f" {_row(row)}") + + # From here on, print changes as they arrive. + await drasi.add_python_reaction("console", query_ids, on_change) + + print( + f"\n=== Watching for changes (Ctrl+C to stop) ===\n" + f"Add or delete messages by running SQL against the database " + f"(docker exec ... psql) and watch the queries react. A sender " + f"appears in inactive-people once they have been quiet for " + f"{INACTIVITY_SECONDS}s.\n" + ) + await asyncio.Event().wait() + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\nStopped.") diff --git a/tutorials/getting-started/database/docker-compose.yml b/tutorials/getting-started/database/docker-compose.yml new file mode 100644 index 0000000..f79805b --- /dev/null +++ b/tutorials/getting-started/database/docker-compose.yml @@ -0,0 +1,44 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: getting-started + +services: + postgres: + image: postgres:16-alpine + container_name: getting-started-postgres + environment: + POSTGRES_DB: getting_started + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres_admin + command: + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_replication_slots=10" + - "-c" + - "max_wal_senders=10" + ports: + - "${POSTGRES_HOST_PORT:-5752}:5432" + volumes: + - getting_started_postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d getting_started"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + getting_started_postgres_data: diff --git a/tutorials/getting-started/database/init.sql b/tutorials/getting-started/database/init.sql new file mode 100644 index 0000000..521f51e --- /dev/null +++ b/tutorials/getting-started/database/init.sql @@ -0,0 +1,92 @@ +-- Copyright 2026 The Drasi Authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Getting Started Tutorial Database Schema +-- +-- A single `message` table -- imagine a simple live message feed. The table and +-- columns are lower-case and unquoted so the node label and property names Drasi +-- sees match the Cypher continuous queries, which use (m:message) and m.message, +-- m.sender, m.messageid without any change. + +-- Suppress noisy output during setup. +\set QUIET on +SET client_min_messages = ERROR; + +-- Create a user with replication privileges for CDC. +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'drasi_user') THEN + CREATE USER drasi_user WITH REPLICATION LOGIN PASSWORD 'drasi_password'; + END IF; +END +$$; + +-- Grant permissions on the database. +GRANT CREATE ON DATABASE getting_started TO drasi_user; +GRANT ALL PRIVILEGES ON DATABASE getting_started TO drasi_user; + +-- Drop existing table if it exists. +DROP TABLE IF EXISTS message CASCADE; + +-- message table: one row per message in the feed. +CREATE TABLE message ( + messageid SERIAL PRIMARY KEY, + sender VARCHAR(50) NOT NULL, + message VARCHAR(200) NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Set REPLICA IDENTITY to FULL so change events include every column. +ALTER TABLE message REPLICA IDENTITY FULL; + +-- Ensure drasi_user owns the table. +ALTER TABLE message OWNER TO drasi_user; + +-- Grant permissions to drasi_user. +GRANT USAGE ON SCHEMA public TO drasi_user; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO drasi_user; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO drasi_user; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO drasi_user; + +-- Create the publication for logical replication. The replication slot itself is +-- created by the Drasi PostgreSQL source on startup (with a consistent +-- snapshot), so the rows seeded below are loaded once via bootstrap and not also +-- replayed as change events. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'drasi_getting_started_pub') THEN + CREATE PUBLICATION drasi_getting_started_pub FOR TABLE message; + END IF; +END +$$; + +-- Seed a few messages. One of them is exactly 'Hello World', which the +-- hello-world-from query matches. +INSERT INTO message (sender, message) +SELECT * FROM (VALUES + ('Buzz Lightyear', 'To infinity and beyond!'), + ('Brian Kernighan', 'Hello World'), + ('Ada Lovelace', 'The Analytical Engine weaves algebraic patterns.') +) AS d(sender, message) +WHERE NOT EXISTS (SELECT 1 FROM message); + +-- Summary. +SET client_min_messages = NOTICE; +DO $$ +BEGIN + RAISE NOTICE 'Getting Started database initialized successfully!'; + RAISE NOTICE 'Table: message'; + RAISE NOTICE 'Publication: drasi_getting_started_pub'; +END +$$; diff --git a/tutorials/getting-started/requirements.txt b/tutorials/getting-started/requirements.txt new file mode 100644 index 0000000..52faded --- /dev/null +++ b/tutorials/getting-started/requirements.txt @@ -0,0 +1,8 @@ +# Getting Started tutorial dependencies. +# +# pip install -r requirements.txt +# +# drasi-lib is the Python binding for Drasi's embeddable engine. It downloads the +# Postgres source/bootstrap plugins from ghcr.io at runtime, so the first run +# needs network access. The app only reads and reacts, so it needs nothing else. +drasi-lib>=0.2,<1 diff --git a/tutorials/getting-started/scripts/cleanup.ps1 b/tutorials/getting-started/scripts/cleanup.ps1 new file mode 100644 index 0000000..63f6362 --- /dev/null +++ b/tutorials/getting-started/scripts/cleanup.ps1 @@ -0,0 +1,48 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup Script (Windows) +# Stops the PostgreSQL container. The app runs in the foreground, so stop it +# first with Ctrl+C. Pass -RemoveVolumes to also delete the database volume. + +param( + [switch]$RemoveVolumes +) + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$DatabaseDir = Join-Path $ScriptDir "..\database" + +Write-Host "=== Getting Started - Cleanup ===" +Write-Host "" + +Push-Location $DatabaseDir +try { + if ($RemoveVolumes) { + Write-Host "Removing container and volumes..." + docker compose down -v 2>&1 | Write-Host + } else { + Write-Host "Removing container (keeping volumes)..." + docker compose down 2>&1 | Write-Host + } +} +finally { + Pop-Location +} + +Write-Host "" +Write-Host "=== Cleanup complete! ===" +Write-Host "" +Write-Host "Options:" +Write-Host " scripts/cleanup.ps1 # Stop containers, keep data volumes" +Write-Host " scripts/cleanup.ps1 -RemoveVolumes # Stop containers and remove data volumes" diff --git a/tutorials/getting-started/scripts/cleanup.sh b/tutorials/getting-started/scripts/cleanup.sh new file mode 100755 index 0000000..b6a41bb --- /dev/null +++ b/tutorials/getting-started/scripts/cleanup.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleanup Script +# Removes the PostgreSQL container. The app runs in the foreground, so stop it +# first with Ctrl+C. Pass --volumes to also delete the database volume. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATABASE_DIR="$SCRIPT_DIR/../database" +REMOVE_VOLUMES="${1:-}" + +echo "=== Getting Started - Cleanup ===" +echo + +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null 2>&1; then + COMPOSE_CMD="docker compose" +else + echo "Warning: docker-compose not found, skipping container cleanup" + exit 0 +fi + +echo "Stopping PostgreSQL container..." +cd "$DATABASE_DIR" + +if [ "$REMOVE_VOLUMES" = "--volumes" ] || [ "$REMOVE_VOLUMES" = "-v" ]; then + echo "Removing container and volumes..." + $COMPOSE_CMD down -v +else + echo "Removing container (keeping volumes)..." + $COMPOSE_CMD down +fi + +echo +echo "=== Cleanup complete! ===" +echo +echo "Options:" +echo " $0 # Stop containers, keep data volumes" +echo " $0 --volumes # Stop containers and remove data volumes" diff --git a/tutorials/getting-started/scripts/setup-database.ps1 b/tutorials/getting-started/scripts/setup-database.ps1 new file mode 100644 index 0000000..f284eda --- /dev/null +++ b/tutorials/getting-started/scripts/setup-database.ps1 @@ -0,0 +1,77 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup Database Script (Windows) +# Starts PostgreSQL with logical replication (WAL) enabled for CDC and seeds the +# message feed. + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$DatabaseDir = Join-Path $ScriptDir "..\database" + +Write-Host "=== Getting Started - Database Setup ===" +Write-Host "" + +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + Write-Host "Error: Docker is not installed or not in PATH" + exit 1 +} + +docker info 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Docker daemon is not running" + exit 1 +} + +Push-Location $DatabaseDir +try { + Write-Host "Stopping any existing PostgreSQL container..." + docker compose down -v 2>&1 | Out-Null + + Write-Host "Starting PostgreSQL with WAL replication..." + docker compose up -d 2>&1 | Write-Host + if ($LASTEXITCODE -ne 0) { + Write-Host "Error: failed to start PostgreSQL container" + exit 1 + } + + Write-Host "Waiting for PostgreSQL to be ready..." + $pgReady = $false + for ($i = 1; $i -le 30; $i++) { + docker exec getting-started-postgres pg_isready -h localhost -U postgres -d getting_started 2>&1 | Out-Null + if ($LASTEXITCODE -eq 0) { Write-Host "PostgreSQL is ready!"; $pgReady = $true; break } + Write-Host " Waiting... ($i/30)"; Start-Sleep -Seconds 2 + } + if (-not $pgReady) { + Write-Host "Error: PostgreSQL did not become ready within the timeout." + Write-Host "Check logs with: docker logs getting-started-postgres" + exit 1 + } + + Write-Host "" + Write-Host "Applying schema and seed data..." + Get-Content (Join-Path $DatabaseDir "init.sql") | ` + docker exec -i getting-started-postgres psql -v ON_ERROR_STOP=1 -U postgres -d getting_started + if ($LASTEXITCODE -ne 0) { Write-Host "Error: failed to apply schema and seed data"; exit 1 } + + Write-Host "" + Write-Host "Seeded messages:" + 'SELECT messageid, sender, message FROM message ORDER BY messageid;' | ` + docker exec -i getting-started-postgres psql -U drasi_user -d getting_started + + Write-Host "" + Write-Host "=== Database setup complete! ===" +} +finally { + Pop-Location +} diff --git a/tutorials/getting-started/scripts/setup-database.sh b/tutorials/getting-started/scripts/setup-database.sh new file mode 100755 index 0000000..ea5cf99 --- /dev/null +++ b/tutorials/getting-started/scripts/setup-database.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup Database Script +# Starts PostgreSQL with logical replication (WAL) enabled for CDC and seeds the +# message feed. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATABASE_DIR="$SCRIPT_DIR/../database" + +echo "=== Getting Started - Database Setup ===" +echo + +if ! command -v docker &> /dev/null; then + echo "Error: Docker is not installed or not in PATH" + echo "Please install Docker: https://docs.docker.com/get-docker/" + exit 1 +fi + +if ! docker info &> /dev/null; then + echo "Error: Docker daemon is not running" + echo "Please start Docker and try again" + exit 1 +fi + +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null 2>&1; then + COMPOSE_CMD="docker compose" +else + echo "Error: docker-compose is not installed" + echo "Please install Docker Compose: https://docs.docker.com/compose/install/" + exit 1 +fi + +echo "Using: $COMPOSE_CMD" +echo + +echo "Stopping any existing PostgreSQL container..." +cd "$DATABASE_DIR" +$COMPOSE_CMD down -v 2>/dev/null || true + +echo "Starting PostgreSQL with WAL replication..." +$COMPOSE_CMD up -d + +echo "Waiting for PostgreSQL to be ready..." +POSTGRES_READY=false +for i in $(seq 1 30); do + if docker exec getting-started-postgres pg_isready -h localhost -U postgres -d getting_started &> /dev/null; then + echo "PostgreSQL is ready!" + POSTGRES_READY=true + break + fi + echo " Waiting... ($i/30)" + sleep 2 +done + +if [ "$POSTGRES_READY" != "true" ]; then + echo "Error: PostgreSQL did not become ready within the timeout." + echo "Check logs with: docker logs getting-started-postgres" + exit 1 +fi + +echo +echo "Applying schema and seed data..." +docker exec -i getting-started-postgres \ + psql -v ON_ERROR_STOP=1 -U postgres -d getting_started < "$DATABASE_DIR/init.sql" + +echo +echo "Seeded messages:" +docker exec getting-started-postgres psql -U drasi_user -d getting_started -c \ + "SELECT messageid, sender, message FROM message ORDER BY messageid;" + +echo +echo "=== Database setup complete! ===" +echo " PostgreSQL: localhost:${POSTGRES_HOST_PORT:-5752} / getting_started" diff --git a/tutorials/getting-started/scripts/start-demo.ps1 b/tutorials/getting-started/scripts/start-demo.ps1 new file mode 100644 index 0000000..9c5c6f3 --- /dev/null +++ b/tutorials/getting-started/scripts/start-demo.ps1 @@ -0,0 +1,41 @@ +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start Demo Script (Windows) +# Starts (and seeds) the database, then runs the console app in the foreground. +# Leave it running in this terminal; use a second terminal to add and delete +# messages with SQL (docker exec ... psql), as shown in the tutorial. Press +# Ctrl+C to stop, then run scripts/cleanup.ps1. + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$TutorialDir = Join-Path $ScriptDir ".." + +& (Join-Path $ScriptDir "setup-database.ps1") +if ($LASTEXITCODE -ne 0) { + Write-Host "Database setup failed; not starting the app." + exit 1 +} + +Write-Host "" +Write-Host "Starting the console app (embeds the Drasi engine)..." +Write-Host "On first run it downloads the Drasi plugins from ghcr.io - give it a moment." +Write-Host "" + +Push-Location $TutorialDir +try { + python app.py +} +finally { + Pop-Location +} diff --git a/tutorials/getting-started/scripts/start-demo.sh b/tutorials/getting-started/scripts/start-demo.sh new file mode 100755 index 0000000..3f223a9 --- /dev/null +++ b/tutorials/getting-started/scripts/start-demo.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2026 The Drasi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start Demo Script +# Starts (and seeds) the database, then runs the console app in the foreground. +# The app embeds the Drasi engine and prints query changes. Leave it running in +# this terminal (Terminal 1); use a second terminal to add and delete messages +# with SQL (docker exec ... psql), as shown in the tutorial. Press Ctrl+C to +# stop, then run ./scripts/cleanup.sh. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TUTORIAL_DIR="$SCRIPT_DIR/.." + +bash "$SCRIPT_DIR/setup-database.sh" + +echo +echo "Starting the console app (embeds the Drasi engine)..." +echo "On first run it downloads the Drasi plugins from ghcr.io — give it a moment." +echo + +cd "$TUTORIAL_DIR" +exec python app.py diff --git a/website/README.md b/website/README.md index f570247..af91be1 100644 --- a/website/README.md +++ b/website/README.md @@ -22,7 +22,7 @@ standard build cannot compile the theme's SCSS — and Node for the theme's asse cd website npm install npm --prefix themes/docsy install -npm run serve # http://localhost:1313/drasi-python/ +npm run serve # http://localhost:1313/ ``` To reproduce what CI publishes: diff --git a/website/assets/scss/_styles_project.scss b/website/assets/scss/_styles_project.scss index 0159285..b7f8a34 100644 --- a/website/assets/scss/_styles_project.scss +++ b/website/assets/scss/_styles_project.scss @@ -278,7 +278,8 @@ pre.chroma, // ============================================================================= // CODE BLOCK COPY BUTTON -// Ensures Docsy's click-to-copy button is visible on dark code blocks +// Docsy's click-to-copy button, made always visible (not hover-only) on the +// dark code blocks so it is easy to discover. // ============================================================================= .td-content .highlight { position: relative; @@ -296,18 +297,19 @@ pre.chroma, right: 8px; top: 8px; padding: 4px 8px; - border: none; + border: 1px solid rgba(255, 255, 255, 0.18); border-radius: $border-radius-sm; - background-color: rgba(255, 255, 255, 0.1); - color: rgba(255, 255, 255, 0.6); + background-color: rgba(255, 255, 255, 0.12); + color: rgba(255, 255, 255, 0.75); cursor: pointer; - opacity: 0; + opacity: 0.7; transition: opacity $transition-fast, background-color $transition-fast, color $transition-fast; z-index: 10; &:hover { - background-color: rgba(255, 255, 255, 0.2); - color: rgba(255, 255, 255, 0.9); + opacity: 1; + background-color: rgba(255, 255, 255, 0.22); + color: rgba(255, 255, 255, 0.95); } &:active { @@ -316,6 +318,7 @@ pre.chroma, } } + // Full strength when the reader is hovering the block. &:hover pre button.td-click-to-copy { opacity: 1; } diff --git a/website/config.toml b/website/config.toml index 9be4846..0a8da6a 100644 --- a/website/config.toml +++ b/website/config.toml @@ -57,6 +57,31 @@ defaultContentLanguageInSubdir = false extended = true min = "0.110.0" + # Tutorials are authored once under ../tutorials/<name>/_index.md (co-located + # with their runnable code) and mounted into the docs content tree, so the doc + # site and the generated GitHub READMEs stay in sync. Adding a content mount + # disables Hugo's implicit "content -> content" default, so it is re-declared + # here; the assets/ and layouts/ defaults are untouched. + [[module.mounts]] + source = "content" + target = "content" + + [[module.mounts]] + source = "../tutorials/getting-started" + target = "content/docs/tutorials/getting-started" + includeFiles = ["/_index.md", "/images/**"] + + [[module.mounts]] + source = "../tutorials/building-comfort" + target = "content/docs/tutorials/building-comfort" + # Only the write-up and its images are docs; the app code is not. + includeFiles = ["/_index.md", "/images/**"] + + [[module.mounts]] + source = "../tutorials/curbside-pickup" + target = "content/docs/tutorials/curbside-pickup" + includeFiles = ["/_index.md", "/images/**"] + # UI customization [params] # Built-in (offline) lunr.js search — no external service required. diff --git a/website/content/docs/tutorials/_index.md b/website/content/docs/tutorials/_index.md new file mode 100644 index 0000000..4a86dd1 --- /dev/null +++ b/website/content/docs/tutorials/_index.md @@ -0,0 +1,45 @@ +--- +title: "Tutorials" +linkTitle: "Tutorials" +weight: 45 +description: > + End-to-end, hands-on walkthroughs you can run and drive yourself. +--- + +Longer, hands-on tutorials that build a complete demo with `drasi-lib` — a real +data source, continuous queries, and a reaction you can watch react in real time. + +Each tutorial is authored once under +[`tutorials/`](https://github.com/drasi-project/drasi-python/tree/main/tutorials) +in the repository (runnable code plus its write-up), so you can read it here or +clone it and run it locally. + +<div class="card-grid card-grid--2"> + <a href="getting-started/"> + <div class="unified-card unified-card--tutorials"> + <div class="unified-card-icon"><i class="fas fa-terminal"></i></div> + <div class="unified-card-content"> + <h3 class="unified-card-title">Getting Started</h3> + <p class="unified-card-summary">Connect Drasi to a live PostgreSQL feed and build three continuous queries (filter, aggregate, and a temporal absence-of-change) in a small no-UI console app driven by a Python reaction.</p> + </div> + </div> + </a> + <a href="building-comfort/"> + <div class="unified-card unified-card--tutorials"> + <div class="unified-card-icon"><i class="fas fa-building"></i></div> + <div class="unified-card-content"> + <h3 class="unified-card-title">Building Comfort</h3> + <p class="unified-card-summary">Monitor a smart building in real time with PostgreSQL CDC, six continuous queries with synthetic joins, and a Python reaction that drives a Streamlit UI.</p> + </div> + </div> + </a> + <a href="curbside-pickup/"> + <div class="unified-card unified-card--tutorials"> + <div class="unified-card-icon"><i class="fas fa-car"></i></div> + <div class="unified-card-content"> + <h3 class="unified-card-title">Curbside Pickup</h3> + <p class="unified-card-summary">Join a PostgreSQL orders store and a MySQL vehicles store by license plate — including a temporal delay query — in one integrated Streamlit UI driven by a Python reaction.</p> + </div> + </div> + </a> +</div> diff --git a/website/layouts/_default/_markup/render-codeblock-cypher.html b/website/layouts/_default/_markup/render-codeblock-cypher.html new file mode 100644 index 0000000..9e9325b --- /dev/null +++ b/website/layouts/_default/_markup/render-codeblock-cypher.html @@ -0,0 +1,30 @@ +{{- /* + Code-block render hook for Cypher. + + Chroma (the highlighter Hugo bundles) has no Cypher lexer, so ```cypher blocks + would otherwise render unhighlighted on the doc site. This hook applies a + lightweight, Cypher-aware highlight using the same Chroma CSS classes + (k, nf, nl, m, c1) the rest of the site already styles. The GitHub READMEs are + generated separately and keep their ```cypher fences, where GitHub's own + Linguist grammar highlights them. +*/ -}} +{{- $h := trim .Inner "\n" | htmlEscape -}} +{{- /* htmlEscape turns ' into the entity '; its digits would otherwise be + caught by the number rule below (producing &#<span...>39</span>; which + browsers can't decode, showing a literal '). Park it on a placeholder + and restore it after all the highlighting regexes have run. */ -}} +{{- $sq := "\ue000" -}} +{{- $h = replace $h "'" $sq -}} +{{- /* line comments */ -}} +{{- $h = $h | replaceRE `(//[^\n]*)` `<span class="c1">$1</span>` -}} +{{- /* keywords */ -}} +{{- $h = $h | replaceRE `(?i)\b(MATCH|OPTIONAL|WITH|WHERE|RETURN|MERGE|CREATE|DELETE|SET|REMOVE|UNWIND|ORDER|BY|SKIP|LIMIT|AS|CASE|WHEN|THEN|ELSE|END|AND|OR|XOR|NOT|IN|STARTS|ENDS|CONTAINS|IS|NULL|TRUE|FALSE|DISTINCT)\b` `<span class="k">$1</span>` -}} +{{- /* functions called as name( — includes Drasi's Cypher extension functions */ -}} +{{- $h = $h | replaceRE `\b(avg|count|sum|min|max|collect|floor|ceil|round|abs|coalesce|toInteger|toFloat|size|length|duration|trueFor|trueLater|changeDateTime|getVersionByTimestamp|realtime|listMax|listMin)(\s*\()` `<span class="nf">$1</span>$2` -}} +{{- /* node / relationship labels after a colon */ -}} +{{- $h = $h | replaceRE `:([A-Za-z_][A-Za-z0-9_]*)` `:<span class="nl">$1</span>` -}} +{{- /* numbers */ -}} +{{- $h = $h | replaceRE `\b(\d+(?:\.\d+)?)\b` `<span class="m">$1</span>` -}} +{{- /* restore the single-quote entity */ -}} +{{- $h = replace $h $sq "'" -}} +<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-cypher" data-lang="cypher">{{ $h | safeHTML }}</code></pre></div> diff --git a/website/package.json b/website/package.json index 5255edb..7118ce5 100644 --- a/website/package.json +++ b/website/package.json @@ -12,7 +12,7 @@ "scripts": { "get:theme": "cd themes/docsy && npm install", "build": "hugo --gc --minify", - "serve": "hugo server" + "serve": "hugo server --baseURL http://localhost:1313/ --appendPort=false" }, "devDependencies": { "autoprefixer": "^10.4.22",