diff --git a/.github/workflows/fetch-telemetry.yml b/.github/workflows/fetch-telemetry.yml new file mode 100644 index 00000000..b2df631f --- /dev/null +++ b/.github/workflows/fetch-telemetry.yml @@ -0,0 +1,43 @@ +name: Fetch Telemetry Data + +on: + schedule: + # Run daily at 08:00 UTC (00:00 Pacific during PST, 01:00 during PDT) + - cron: '0 8 * * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + fetch-telemetry: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Fetch and transform telemetry data + run: python3 hack/fetch_telemetry.py + + - name: Check for changes + id: changes + run: | + if git diff --quiet static/oss-health-data/telemetry.json; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push + if: steps.changes.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add static/oss-health-data/telemetry.json + git commit -m "chore(oss-health): update telemetry snapshot" + git push diff --git a/assets/scss/_oss_health.scss b/assets/scss/_oss_health.scss index b9398549..649187e4 100644 --- a/assets/scss/_oss_health.scss +++ b/assets/scss/_oss_health.scss @@ -228,6 +228,13 @@ font-size: clamp(2rem, 3vw, 2.8rem); line-height: 1; } + + &__hint { + color: #637595; + display: block; + font-size: 0.85rem; + margin-top: 0.6rem; + } } .oss-health-mini { diff --git a/assets/scss/_telemetry.scss b/assets/scss/_telemetry.scss new file mode 100644 index 00000000..2b48298d --- /dev/null +++ b/assets/scss/_telemetry.scss @@ -0,0 +1,90 @@ +/* telemetry page */ + +.telemetry-page { + margin-top: 4rem; + + @include media-breakpoint-down(sm) { + margin-top: 2rem; + } + + .nav-tabs { + border-bottom: 2px solid $primary; + + .nav-link { + color: $cozy-mid-gray; + font-weight: 600; + border: none; + border-bottom: 3px solid transparent; + padding: 0.75rem 1.5rem; + + &:hover { + color: $primary; + border-bottom-color: rgba($primary, 0.3); + } + + &.active { + color: $primary; + border-bottom-color: $primary; + background: transparent; + } + } + } + + .telemetry-card { + border: none; + border-radius: 0.75rem; + transition: transform 0.15s ease; + + &:hover { + transform: translateY(-2px); + } + + .telemetry-icon { + font-size: 1.75rem; + color: $primary; + margin-bottom: 0.5rem; + } + + .telemetry-value { + font-size: 2.5rem; + font-weight: 700; + color: $cozy-black; + line-height: 1.2; + } + + .telemetry-label { + font-size: 0.95rem; + font-weight: 600; + color: $cozy-mid-gray; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-top: 0.25rem; + } + + .telemetry-secondary { + font-size: 0.85rem; + color: $cozy-light-gray; + margin-top: 0.25rem; + } + } + + .table { + th { + font-weight: 600; + } + + .table-primary { + --bs-table-bg: #{rgba($primary, 0.08)}; + --bs-table-border-color: #{rgba($primary, 0.15)}; + color: $cozy-black; + } + + code { + color: $primary; + font-weight: 500; + background: rgba($primary, 0.06); + padding: 0.15rem 0.4rem; + border-radius: 0.25rem; + } + } +} diff --git a/assets/scss/main.scss b/assets/scss/main.scss index 1e633f9f..e91fab0b 100644 --- a/assets/scss/main.scss +++ b/assets/scss/main.scss @@ -161,3 +161,4 @@ a { @import "announcement-banner"; @import "tabs_alerts"; @import "override-docsy-tabs"; +@import "telemetry"; diff --git a/content/en/oss-health/_index.md b/content/en/oss-health/_index.md index bb2dcec5..1cc2f586 100644 --- a/content/en/oss-health/_index.md +++ b/content/en/oss-health/_index.md @@ -1,6 +1,7 @@ --- title: "OSS Health" description: "Open source project health snapshots for Cozystack." +type: oss-health --- Project health snapshots for Cozystack across community activity, repository trends, and security posture. diff --git a/content/en/oss-health/telemetry.md b/content/en/oss-health/telemetry.md new file mode 100644 index 00000000..37c96e1b --- /dev/null +++ b/content/en/oss-health/telemetry.md @@ -0,0 +1,10 @@ +--- +title: "Telemetry" +layout: "oss-health-app" +draft: false +description: "Anonymous usage statistics reported by live Cozystack clusters." +oss_health_key: "telemetry" +oss_health_kind: "telemetry" +source_url: "https://telemetry.cozystack.io/" +lede: "Monthly, quarterly, and yearly snapshots of Cozystack fleet size and application usage across opted-in installations." +--- diff --git a/hack/fetch_telemetry.py b/hack/fetch_telemetry.py new file mode 100755 index 00000000..54afeb0c --- /dev/null +++ b/hack/fetch_telemetry.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +Fetch Cozystack telemetry and produce a JSON payload for the OSS Health shell. + +What it does: +1. Query https://telemetry.cozystack.io/api/overview?year=YYYY&month=MM +2. Filter apps to entries visible on the Cozystack dashboard. +3. Merge case-insensitive / Pax* / legacy-name aliases into one canonical entry + per application, keeping the maximum instance count (zero-count entries + left after the merge are dropped from the table). +4. Pull `Tenant` out of the apps map and surface it as the top-level Tenants + summary card (the raw `total_tenants` field from the API is always zero). +5. Emit the payload in the shape consumed by `oss-health-app.html` + + `renderTelemetry`, including `summary_cards`, `apps`, `range`. + +Used by both `.github/workflows/fetch-telemetry.yml` (daily cron) and the +developer who needs to refresh the seed file locally. +""" +from __future__ import annotations + +import datetime as dt +import json +import os +import sys +import urllib.error +import urllib.request + +API_URL = "https://telemetry.cozystack.io/api/overview" +OUTPUT_PATH = os.environ.get( + "TELEMETRY_OUTPUT_PATH", + "static/oss-health-data/telemetry.json", +) + +# Canonical display name per normalized key. Anything not in this table is +# dropped (internal entities like `Info`, `Pax*` experimental variants that +# don't map to a dashboard app, duplicate lowercase CR kind names, etc.). +# Keys are lower-cased and stripped of hyphens so we can match PascalCase, +# lowercase, kebab-case and Pax-prefixed variants against the same canonical. +ALIASES: dict[str, str] = { + # Managed applications (docs/v1.2/applications/_include/*) + "clickhouse": "ClickHouse", + "paxclickhouse": "ClickHouse", + "foundationdb": "FoundationDB", + "harbor": "Harbor", + "kafka": "Kafka", + "mariadb": "MariaDB", + "mongodb": "MongoDB", + "nats": "NATS", + "openbao": "OpenBAO", + "opensearch": "OpenSearch", + "postgres": "Postgres", + "postgresql": "Postgres", + "paxpostgres": "Postgres", + "qdrant": "Qdrant", + "rabbitmq": "RabbitMQ", + "redis": "Redis", + "paxredis": "Redis", + "clearml": "ClearML", + # Services (docs/v1.2/operations/services/*) + "etcd": "Etcd", + "ingress": "Ingress", + "monitoring": "Monitoring", + "bucket": "Bucket", + "seaweedfs": "SeaweedFS", + "nfs": "NFS", + # Networking (docs/v1.2/networking/_include/*) + "httpcache": "HTTPCache", + "tcpbalancer": "TCPBalancer", + "virtualprivatecloud": "VirtualPrivateCloud", + "vpc": "VirtualPrivateCloud", + "vpn": "VPN", + # Virtualization (docs/v1.2/virtualization/_include/*) + "vminstance": "VMInstance", + "paxvminstance": "VMInstance", + "vmdisk": "VMDisk", + # Managed Kubernetes + "kubernetes": "Kubernetes", +} + + +def normalize_key(raw: str) -> str: + return raw.lower().replace("-", "").replace("_", "") + + +def clean_apps(apps: dict[str, int]) -> list[dict[str, object]]: + """Filter, dedupe (max), drop zeros, sort desc by count.""" + merged: dict[str, int] = {} + for raw_name, count in apps.items(): + canonical = ALIASES.get(normalize_key(raw_name)) + if not canonical: + continue + if count > merged.get(canonical, 0): + merged[canonical] = count + non_zero = [(name, n) for name, n in merged.items() if n > 0] + non_zero.sort(key=lambda item: (-item[1], item[0].lower())) + return [{"name": name, "value": str(count)} for name, count in non_zero] + + +def transform_period(raw_period: dict, label_fallback: str) -> dict | None: + if not raw_period: + return None + apps_raw = raw_period.get("apps", {}) or {} + tenants = int(apps_raw.get("Tenant", 0)) + clusters = int(raw_period.get("clusters", 0)) + total_nodes = int(raw_period.get("total_nodes", 0)) + avg_nodes = raw_period.get("avg_nodes_per_cluster") + summary = [ + {"label": "Clusters", "value": str(clusters)}, + { + "label": "Total Nodes", + "value": str(total_nodes), + "hint": ( + f"avg {avg_nodes:.1f} per cluster" + if clusters and isinstance(avg_nodes, (int, float)) + else "" + ), + }, + { + "label": "Tenants", + "value": str(tenants), + "hint": ( + f"avg {tenants / clusters:.1f} per cluster" + if clusters + else "" + ), + }, + ] + period = { + "label": raw_period.get("label") or label_fallback, + "summary_cards": summary, + "apps": clean_apps(apps_raw), + } + start = raw_period.get("start") + end = raw_period.get("end") + if start and end: + period["range"] = {"from": start, "to": end} + return period + + +def fetch(year: int, month: int) -> dict: + url = f"{API_URL}?year={year}&month={month:02d}" + req = urllib.request.Request(url, headers={"User-Agent": "cozystack-website/telemetry-fetch"}) + with urllib.request.urlopen(req, timeout=30) as resp: + if resp.status != 200: + raise RuntimeError(f"telemetry API returned HTTP {resp.status}") + return json.loads(resp.read().decode("utf-8")) + + +def build_payload(raw: dict) -> dict: + periods_raw = raw.get("periods", {}) or {} + periods_out: dict[str, dict] = {} + for key in ("month", "quarter", "year"): + transformed = transform_period(periods_raw.get(key, {}) or {}, label_fallback=key.title()) + if transformed: + periods_out[key] = transformed + return { + "updated_at": raw.get("generated_at"), + "title": "Telemetry", + "source": {"label": "Cozystack Telemetry Server"}, + "periods": periods_out, + } + + +def main() -> int: + today = dt.datetime.now(dt.timezone.utc) + year = int(os.environ.get("TELEMETRY_YEAR", today.year)) + month = int(os.environ.get("TELEMETRY_MONTH", today.month)) + try: + raw = fetch(year, month) + except (urllib.error.URLError, urllib.error.HTTPError, RuntimeError, ValueError) as err: + print(f"fetch failed: {err}", file=sys.stderr) + return 1 + payload = build_payload(raw) + if not payload["periods"]: + print("fetched payload has no usable periods; refusing to write empty file", file=sys.stderr) + return 1 + os.makedirs(os.path.dirname(OUTPUT_PATH), exist_ok=True) + with open(OUTPUT_PATH, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, ensure_ascii=False) + fh.write("\n") + print(f"wrote {OUTPUT_PATH} ({len(payload['periods'])} periods, {sum(len(p['apps']) for p in payload['periods'].values())} app rows total)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hugo.yaml b/hugo.yaml index 6bd7bf90..1d424cf7 100644 --- a/hugo.yaml +++ b/hugo.yaml @@ -206,18 +206,22 @@ menus: - name: OSS Health identifier: oss-health weight: 35 + - name: Telemetry + url: /oss-health/telemetry/ + parent: oss-health + weight: 1 - name: DevStats url: /oss-health/devstats/ parent: oss-health - weight: 1 + weight: 2 - name: OpenSSF url: /oss-health/openssf/ parent: oss-health - weight: 2 + weight: 3 - name: OSS Insight url: /oss-health/oss-insight/ parent: oss-health - weight: 3 + weight: 4 - name: Enterprise support url: /support weight: 5 diff --git a/layouts/_default/oss-health-app.html b/layouts/_default/oss-health-app.html index a85aa981..4fc25b32 100644 --- a/layouts/_default/oss-health-app.html +++ b/layouts/_default/oss-health-app.html @@ -80,6 +80,7 @@

{{ .Title }}

${escapeHtml(card.label)} ${escapeHtml(card.value)} + ${card.hint ? `${escapeHtml(card.hint)}` : ""}
`).join("")} @@ -200,6 +201,72 @@

${escapeHtml(payload.title)} snapshot

renderActive(); }; + const renderTelemetry = (payload) => { + const order = ["month", "quarter", "year"]; + const available = order.filter((name) => payload.periods[name]); + if (!available.length) { + root.innerHTML = `

No telemetry data available yet.

`; + return; + } + let active = available[0]; + + const renderActive = () => { + const period = payload.periods[active]; + const controls = ` +
+ ${available.map((name) => ` + + `).join("")} +
+ `; + + const rangeStrip = period.range + ? `
+
+ Window + ${escapeHtml(period.label)} +
+
+ Range + ${escapeHtml(period.range.from)} — ${escapeHtml(period.range.to)} +
+
` + : ""; + + const appsSection = period.apps && period.apps.length + ? renderTable("Applications in use", "Instances", period.apps) + : `

No application instances reported for this period yet.

`; + + root.innerHTML = ` +
+
+
+

${escapeHtml(payload.source.label)}

+

${escapeHtml(payload.title)} snapshot

+
+ ${controls} +
+
+ ${renderCards(period.summary_cards)} + ${rangeStrip} + ${appsSection} +
+
+ `; + + root.querySelectorAll("[data-period]").forEach((button) => { + button.addEventListener("click", () => { + active = button.dataset.period; + renderActive(); + }); + }); + }; + + renderActive(); + }; + const renderOpenSSF = (payload) => { const badgeUpdated = payload.badge_last_updated_at ? `
Badge updated${escapeHtml(formatDateTime(payload.badge_last_updated_at))}
` @@ -252,6 +319,8 @@

${escapeHtml(payload.project_name)}

updatedNode.textContent = `Updated ${formatDateTime(payload.updated_at)}`; if (kind === "state") { renderOpenSSF(payload); + } else if (kind === "telemetry") { + renderTelemetry(payload); } else { renderTimeseries(payload); } diff --git a/layouts/oss-health/baseof.html b/layouts/oss-health/baseof.html new file mode 100644 index 00000000..d22e0c7e --- /dev/null +++ b/layouts/oss-health/baseof.html @@ -0,0 +1,18 @@ + + + + {{ partial "head.html" . }} + + +
+ {{ partial "navbar.html" . }} +
+
+
+ {{ block "main" . }}{{ end }} +
+ {{ partial "footer.html" . }} +
+ {{ partial "scripts.html" . }} + + diff --git a/static/oss-health-data/telemetry.json b/static/oss-health-data/telemetry.json new file mode 100644 index 00000000..b7d234c2 --- /dev/null +++ b/static/oss-health-data/telemetry.json @@ -0,0 +1,336 @@ +{ + "updated_at": "2026-04-17T19:38:08Z", + "title": "Telemetry", + "source": { + "label": "Cozystack Telemetry Server" + }, + "periods": { + "month": { + "label": "April 2026", + "summary_cards": [ + { + "label": "Clusters", + "value": "43" + }, + { + "label": "Total Nodes", + "value": "164", + "hint": "avg 3.8 per cluster" + }, + { + "label": "Tenants", + "value": "83", + "hint": "avg 1.9 per cluster" + } + ], + "apps": [ + { + "name": "Ingress", + "value": "44" + }, + { + "name": "Etcd", + "value": "40" + }, + { + "name": "Kubernetes", + "value": "30" + }, + { + "name": "Monitoring", + "value": "29" + }, + { + "name": "Bucket", + "value": "21" + }, + { + "name": "SeaweedFS", + "value": "14" + }, + { + "name": "VMInstance", + "value": "12" + }, + { + "name": "VMDisk", + "value": "11" + }, + { + "name": "VirtualPrivateCloud", + "value": "7" + }, + { + "name": "Postgres", + "value": "6" + }, + { + "name": "Redis", + "value": "5" + }, + { + "name": "MongoDB", + "value": "2" + }, + { + "name": "Harbor", + "value": "1" + }, + { + "name": "Kafka", + "value": "1" + }, + { + "name": "MariaDB", + "value": "1" + }, + { + "name": "NFS", + "value": "1" + }, + { + "name": "OpenBAO", + "value": "1" + }, + { + "name": "RabbitMQ", + "value": "1" + }, + { + "name": "TCPBalancer", + "value": "1" + } + ], + "range": { + "from": "2026-04-01", + "to": "2026-04-30" + } + }, + "quarter": { + "label": "March 2026 — April 2026", + "summary_cards": [ + { + "label": "Clusters", + "value": "41" + }, + { + "label": "Total Nodes", + "value": "162", + "hint": "avg 4.0 per cluster" + }, + { + "label": "Tenants", + "value": "55", + "hint": "avg 1.3 per cluster" + } + ], + "apps": [ + { + "name": "Ingress", + "value": "32" + }, + { + "name": "Etcd", + "value": "26" + }, + { + "name": "Monitoring", + "value": "23" + }, + { + "name": "Kubernetes", + "value": "18" + }, + { + "name": "Bucket", + "value": "15" + }, + { + "name": "SeaweedFS", + "value": "10" + }, + { + "name": "VMDisk", + "value": "8" + }, + { + "name": "VMInstance", + "value": "7" + }, + { + "name": "Postgres", + "value": "6" + }, + { + "name": "Redis", + "value": "5" + }, + { + "name": "VirtualPrivateCloud", + "value": "4" + }, + { + "name": "Qdrant", + "value": "2" + }, + { + "name": "ClickHouse", + "value": "1" + }, + { + "name": "Harbor", + "value": "1" + }, + { + "name": "Kafka", + "value": "1" + }, + { + "name": "MariaDB", + "value": "1" + }, + { + "name": "MongoDB", + "value": "1" + }, + { + "name": "NATS", + "value": "1" + }, + { + "name": "NFS", + "value": "1" + }, + { + "name": "OpenBAO", + "value": "1" + }, + { + "name": "RabbitMQ", + "value": "1" + }, + { + "name": "TCPBalancer", + "value": "1" + } + ], + "range": { + "from": "2026-03-01", + "to": "2026-04-30" + } + }, + "year": { + "label": "March 2026 — April 2026", + "summary_cards": [ + { + "label": "Clusters", + "value": "41" + }, + { + "label": "Total Nodes", + "value": "162", + "hint": "avg 4.0 per cluster" + }, + { + "label": "Tenants", + "value": "55", + "hint": "avg 1.3 per cluster" + } + ], + "apps": [ + { + "name": "Ingress", + "value": "32" + }, + { + "name": "Etcd", + "value": "26" + }, + { + "name": "Monitoring", + "value": "23" + }, + { + "name": "Kubernetes", + "value": "18" + }, + { + "name": "Bucket", + "value": "15" + }, + { + "name": "SeaweedFS", + "value": "10" + }, + { + "name": "VMDisk", + "value": "8" + }, + { + "name": "VMInstance", + "value": "7" + }, + { + "name": "Postgres", + "value": "6" + }, + { + "name": "Redis", + "value": "5" + }, + { + "name": "VirtualPrivateCloud", + "value": "4" + }, + { + "name": "Qdrant", + "value": "2" + }, + { + "name": "ClickHouse", + "value": "1" + }, + { + "name": "Harbor", + "value": "1" + }, + { + "name": "Kafka", + "value": "1" + }, + { + "name": "MariaDB", + "value": "1" + }, + { + "name": "MongoDB", + "value": "1" + }, + { + "name": "NATS", + "value": "1" + }, + { + "name": "NFS", + "value": "1" + }, + { + "name": "OpenBAO", + "value": "1" + }, + { + "name": "RabbitMQ", + "value": "1" + }, + { + "name": "TCPBalancer", + "value": "1" + } + ], + "range": { + "from": "2026-03-01", + "to": "2026-04-30" + } + } + } +}