diff --git a/.github/workflows/fetch-telemetry.yml b/.github/workflows/fetch-telemetry.yml new file mode 100644 index 00000000..b2df631f --- /dev/null +++ b/.github/workflows/fetch-telemetry.yml @@ -0,0 +1,43 @@ +name: Fetch Telemetry Data + +on: + schedule: + # Run daily at 08:00 UTC (00:00 Pacific during PST, 01:00 during PDT) + - cron: '0 8 * * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + fetch-telemetry: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Fetch and transform telemetry data + run: python3 hack/fetch_telemetry.py + + - name: Check for changes + id: changes + run: | + if git diff --quiet static/oss-health-data/telemetry.json; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push + if: steps.changes.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add static/oss-health-data/telemetry.json + git commit -m "chore(oss-health): update telemetry snapshot" + git push diff --git a/assets/scss/_oss_health.scss b/assets/scss/_oss_health.scss index b9398549..649187e4 100644 --- a/assets/scss/_oss_health.scss +++ b/assets/scss/_oss_health.scss @@ -228,6 +228,13 @@ font-size: clamp(2rem, 3vw, 2.8rem); line-height: 1; } + + &__hint { + color: #637595; + display: block; + font-size: 0.85rem; + margin-top: 0.6rem; + } } .oss-health-mini { diff --git a/assets/scss/_telemetry.scss b/assets/scss/_telemetry.scss new file mode 100644 index 00000000..2b48298d --- /dev/null +++ b/assets/scss/_telemetry.scss @@ -0,0 +1,90 @@ +/* telemetry page */ + +.telemetry-page { + margin-top: 4rem; + + @include media-breakpoint-down(sm) { + margin-top: 2rem; + } + + .nav-tabs { + border-bottom: 2px solid $primary; + + .nav-link { + color: $cozy-mid-gray; + font-weight: 600; + border: none; + border-bottom: 3px solid transparent; + padding: 0.75rem 1.5rem; + + &:hover { + color: $primary; + border-bottom-color: rgba($primary, 0.3); + } + + &.active { + color: $primary; + border-bottom-color: $primary; + background: transparent; + } + } + } + + .telemetry-card { + border: none; + border-radius: 0.75rem; + transition: transform 0.15s ease; + + &:hover { + transform: translateY(-2px); + } + + .telemetry-icon { + font-size: 1.75rem; + color: $primary; + margin-bottom: 0.5rem; + } + + .telemetry-value { + font-size: 2.5rem; + font-weight: 700; + color: $cozy-black; + line-height: 1.2; + } + + .telemetry-label { + font-size: 0.95rem; + font-weight: 600; + color: $cozy-mid-gray; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-top: 0.25rem; + } + + .telemetry-secondary { + font-size: 0.85rem; + color: $cozy-light-gray; + margin-top: 0.25rem; + } + } + + .table { + th { + font-weight: 600; + } + + .table-primary { + --bs-table-bg: #{rgba($primary, 0.08)}; + --bs-table-border-color: #{rgba($primary, 0.15)}; + color: $cozy-black; + } + + code { + color: $primary; + font-weight: 500; + background: rgba($primary, 0.06); + padding: 0.15rem 0.4rem; + border-radius: 0.25rem; + } + } +} diff --git a/assets/scss/main.scss b/assets/scss/main.scss index 1e633f9f..e91fab0b 100644 --- a/assets/scss/main.scss +++ b/assets/scss/main.scss @@ -161,3 +161,4 @@ a { @import "announcement-banner"; @import "tabs_alerts"; @import "override-docsy-tabs"; +@import "telemetry"; diff --git a/content/en/oss-health/_index.md b/content/en/oss-health/_index.md index bb2dcec5..1cc2f586 100644 --- a/content/en/oss-health/_index.md +++ b/content/en/oss-health/_index.md @@ -1,6 +1,7 @@ --- title: "OSS Health" description: "Open source project health snapshots for Cozystack." +type: oss-health --- Project health snapshots for Cozystack across community activity, repository trends, and security posture. diff --git a/content/en/oss-health/telemetry.md b/content/en/oss-health/telemetry.md new file mode 100644 index 00000000..37c96e1b --- /dev/null +++ b/content/en/oss-health/telemetry.md @@ -0,0 +1,10 @@ +--- +title: "Telemetry" +layout: "oss-health-app" +draft: false +description: "Anonymous usage statistics reported by live Cozystack clusters." +oss_health_key: "telemetry" +oss_health_kind: "telemetry" +source_url: "https://telemetry.cozystack.io/" +lede: "Monthly, quarterly, and yearly snapshots of Cozystack fleet size and application usage across opted-in installations." +--- diff --git a/hack/fetch_telemetry.py b/hack/fetch_telemetry.py new file mode 100755 index 00000000..54afeb0c --- /dev/null +++ b/hack/fetch_telemetry.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +Fetch Cozystack telemetry and produce a JSON payload for the OSS Health shell. + +What it does: +1. Query https://telemetry.cozystack.io/api/overview?year=YYYY&month=MM +2. Filter apps to entries visible on the Cozystack dashboard. +3. Merge case-insensitive / Pax* / legacy-name aliases into one canonical entry + per application, keeping the maximum instance count (zero-count entries + left after the merge are dropped from the table). +4. Pull `Tenant` out of the apps map and surface it as the top-level Tenants + summary card (the raw `total_tenants` field from the API is always zero). +5. Emit the payload in the shape consumed by `oss-health-app.html` + + `renderTelemetry`, including `summary_cards`, `apps`, `range`. + +Used by both `.github/workflows/fetch-telemetry.yml` (daily cron) and the +developer who needs to refresh the seed file locally. +""" +from __future__ import annotations + +import datetime as dt +import json +import os +import sys +import urllib.error +import urllib.request + +API_URL = "https://telemetry.cozystack.io/api/overview" +OUTPUT_PATH = os.environ.get( + "TELEMETRY_OUTPUT_PATH", + "static/oss-health-data/telemetry.json", +) + +# Canonical display name per normalized key. Anything not in this table is +# dropped (internal entities like `Info`, `Pax*` experimental variants that +# don't map to a dashboard app, duplicate lowercase CR kind names, etc.). +# Keys are lower-cased and stripped of hyphens so we can match PascalCase, +# lowercase, kebab-case and Pax-prefixed variants against the same canonical. +ALIASES: dict[str, str] = { + # Managed applications (docs/v1.2/applications/_include/*) + "clickhouse": "ClickHouse", + "paxclickhouse": "ClickHouse", + "foundationdb": "FoundationDB", + "harbor": "Harbor", + "kafka": "Kafka", + "mariadb": "MariaDB", + "mongodb": "MongoDB", + "nats": "NATS", + "openbao": "OpenBAO", + "opensearch": "OpenSearch", + "postgres": "Postgres", + "postgresql": "Postgres", + "paxpostgres": "Postgres", + "qdrant": "Qdrant", + "rabbitmq": "RabbitMQ", + "redis": "Redis", + "paxredis": "Redis", + "clearml": "ClearML", + # Services (docs/v1.2/operations/services/*) + "etcd": "Etcd", + "ingress": "Ingress", + "monitoring": "Monitoring", + "bucket": "Bucket", + "seaweedfs": "SeaweedFS", + "nfs": "NFS", + # Networking (docs/v1.2/networking/_include/*) + "httpcache": "HTTPCache", + "tcpbalancer": "TCPBalancer", + "virtualprivatecloud": "VirtualPrivateCloud", + "vpc": "VirtualPrivateCloud", + "vpn": "VPN", + # Virtualization (docs/v1.2/virtualization/_include/*) + "vminstance": "VMInstance", + "paxvminstance": "VMInstance", + "vmdisk": "VMDisk", + # Managed Kubernetes + "kubernetes": "Kubernetes", +} + + +def normalize_key(raw: str) -> str: + return raw.lower().replace("-", "").replace("_", "") + + +def clean_apps(apps: dict[str, int]) -> list[dict[str, object]]: + """Filter, dedupe (max), drop zeros, sort desc by count.""" + merged: dict[str, int] = {} + for raw_name, count in apps.items(): + canonical = ALIASES.get(normalize_key(raw_name)) + if not canonical: + continue + if count > merged.get(canonical, 0): + merged[canonical] = count + non_zero = [(name, n) for name, n in merged.items() if n > 0] + non_zero.sort(key=lambda item: (-item[1], item[0].lower())) + return [{"name": name, "value": str(count)} for name, count in non_zero] + + +def transform_period(raw_period: dict, label_fallback: str) -> dict | None: + if not raw_period: + return None + apps_raw = raw_period.get("apps", {}) or {} + tenants = int(apps_raw.get("Tenant", 0)) + clusters = int(raw_period.get("clusters", 0)) + total_nodes = int(raw_period.get("total_nodes", 0)) + avg_nodes = raw_period.get("avg_nodes_per_cluster") + summary = [ + {"label": "Clusters", "value": str(clusters)}, + { + "label": "Total Nodes", + "value": str(total_nodes), + "hint": ( + f"avg {avg_nodes:.1f} per cluster" + if clusters and isinstance(avg_nodes, (int, float)) + else "" + ), + }, + { + "label": "Tenants", + "value": str(tenants), + "hint": ( + f"avg {tenants / clusters:.1f} per cluster" + if clusters + else "" + ), + }, + ] + period = { + "label": raw_period.get("label") or label_fallback, + "summary_cards": summary, + "apps": clean_apps(apps_raw), + } + start = raw_period.get("start") + end = raw_period.get("end") + if start and end: + period["range"] = {"from": start, "to": end} + return period + + +def fetch(year: int, month: int) -> dict: + url = f"{API_URL}?year={year}&month={month:02d}" + req = urllib.request.Request(url, headers={"User-Agent": "cozystack-website/telemetry-fetch"}) + with urllib.request.urlopen(req, timeout=30) as resp: + if resp.status != 200: + raise RuntimeError(f"telemetry API returned HTTP {resp.status}") + return json.loads(resp.read().decode("utf-8")) + + +def build_payload(raw: dict) -> dict: + periods_raw = raw.get("periods", {}) or {} + periods_out: dict[str, dict] = {} + for key in ("month", "quarter", "year"): + transformed = transform_period(periods_raw.get(key, {}) or {}, label_fallback=key.title()) + if transformed: + periods_out[key] = transformed + return { + "updated_at": raw.get("generated_at"), + "title": "Telemetry", + "source": {"label": "Cozystack Telemetry Server"}, + "periods": periods_out, + } + + +def main() -> int: + today = dt.datetime.now(dt.timezone.utc) + year = int(os.environ.get("TELEMETRY_YEAR", today.year)) + month = int(os.environ.get("TELEMETRY_MONTH", today.month)) + try: + raw = fetch(year, month) + except (urllib.error.URLError, urllib.error.HTTPError, RuntimeError, ValueError) as err: + print(f"fetch failed: {err}", file=sys.stderr) + return 1 + payload = build_payload(raw) + if not payload["periods"]: + print("fetched payload has no usable periods; refusing to write empty file", file=sys.stderr) + return 1 + os.makedirs(os.path.dirname(OUTPUT_PATH), exist_ok=True) + with open(OUTPUT_PATH, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, ensure_ascii=False) + fh.write("\n") + print(f"wrote {OUTPUT_PATH} ({len(payload['periods'])} periods, {sum(len(p['apps']) for p in payload['periods'].values())} app rows total)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hugo.yaml b/hugo.yaml index 6bd7bf90..1d424cf7 100644 --- a/hugo.yaml +++ b/hugo.yaml @@ -206,18 +206,22 @@ menus: - name: OSS Health identifier: oss-health weight: 35 + - name: Telemetry + url: /oss-health/telemetry/ + parent: oss-health + weight: 1 - name: DevStats url: /oss-health/devstats/ parent: oss-health - weight: 1 + weight: 2 - name: OpenSSF url: /oss-health/openssf/ parent: oss-health - weight: 2 + weight: 3 - name: OSS Insight url: /oss-health/oss-insight/ parent: oss-health - weight: 3 + weight: 4 - name: Enterprise support url: /support weight: 5 diff --git a/layouts/_default/oss-health-app.html b/layouts/_default/oss-health-app.html index a85aa981..4fc25b32 100644 --- a/layouts/_default/oss-health-app.html +++ b/layouts/_default/oss-health-app.html @@ -80,6 +80,7 @@
No telemetry data available yet.
`; + return; + } + let active = available[0]; + + const renderActive = () => { + const period = payload.periods[active]; + const controls = ` +No application instances reported for this period yet.
`; + + root.innerHTML = ` +${escapeHtml(payload.source.label)}
+