feat: add cloudflare-metrics worker for graphql analytics export

.github/workflows/build.yml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
       - name: Setup pnpm
-        uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
       - name: Setup Node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
         with:
           node-version-file: '.nvmrc'
       - name: Run pnpm install
@@ -34,7 +34,7 @@ jobs:
         run: pnpm build
 
       - name: Upload build output
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: build-output
           path: 'dist'
@@ -52,18 +52,18 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_TF_DEV_ENV }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Get build artifact
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: 'build-output'
           path: '${{ github.workspace }}/dist'
 
       - name: Install 1Password CLI
-        uses: 1password/install-cli-action@9a0c9dd934086b7ab1d90115d455bda1c53c2bdb # v2.0.2
+        uses: 1password/install-cli-action@8d006a0d0a4fd505af7f7ce589e7f768385ff5e4 # v3.0.0
 
       - name: Setup Mise
         uses: immich-app/devtools/actions/use-mise@697a75e2c3186d3c037c2c159855cf2d566542ba # use-mise-action-0.0.1
@@ -100,7 +100,7 @@ jobs:
 
       - name: Comment preview URLs
         if: ${{ github.event_name == 'pull_request' }}
-        uses: actions-cool/maintain-one-comment@4b2dbf086015f892dcb5e8c1106f5fccd6c1476b # v3.2.0
+        uses: actions-cool/maintain-one-comment@772214ac4bff3c578c76eed4ac19d08dc787c050 # v3.2.1
         with:
           number: ${{ github.event.number }}
           body: ${{ steps.preview-urls.outputs.body }}
@@ -120,18 +120,18 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_TF_PROD_ENV }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Get build artifact
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: 'build-output'
           path: '${{ github.workspace }}/dist'
 
       - name: Install 1Password CLI
-        uses: 1password/install-cli-action@9a0c9dd934086b7ab1d90115d455bda1c53c2bdb # v2.0.2
+        uses: 1password/install-cli-action@8d006a0d0a4fd505af7f7ce589e7f768385ff5e4 # v3.0.0
 
       - name: Setup Mise
         uses: immich-app/devtools/actions/use-mise@697a75e2c3186d3c037c2c159855cf2d566542ba # use-mise-action-0.0.1

.github/workflows/test.yml

@@ -15,11 +15,13 @@
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
       - name: Setup pnpm
-        uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
       - name: Setup Node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
         with:
           node-version-file: '.nvmrc'
 
@@ -41,3 +43,64 @@
       - name: Run unit tests
         run: pnpm run test
         if: ${{ !cancelled() }}
+
+  integration-test-cloudflare-metrics:
+    name: Integration Test (cloudflare-metrics)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        with:
+          filters: |
+            cloudflare-metrics:
+              - 'apps/cloudflare-metrics/**'
+              - '.github/workflows/test.yml'
+
+      - name: Setup pnpm
+        if: steps.changes.outputs.cloudflare-metrics == 'true' || github.event_name != 'pull_request'
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+
+      - name: Setup Node
+        if: steps.changes.outputs.cloudflare-metrics == 'true' || github.event_name != 'pull_request'
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Run pnpm install
+        if: steps.changes.outputs.cloudflare-metrics == 'true' || github.event_name != 'pull_request'
+        run: pnpm install --frozen-lockfile
+
+      - name: Install 1Password CLI
+        if: steps.changes.outputs.cloudflare-metrics == 'true' || github.event_name != 'pull_request'
+        uses: 1password/install-cli-action@8d006a0d0a4fd505af7f7ce589e7f768385ff5e4 # v3.0.0
+
+      - name: Check integration credentials are available
+        id: creds
+        if: steps.changes.outputs.cloudflare-metrics == 'true' || github.event_name != 'pull_request'
+        continue-on-error: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_TF_DEV_ENV }}
+        run: |
+          if op run --env-file=apps/cloudflare-metrics/.integration.env -- sh -c 'test -n "$CLOUDFLARE_API_TOKEN" && test -n "$CLOUDFLARE_ACCOUNT_ID"' 2>/dev/null; then
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "available=false" >> "$GITHUB_OUTPUT"
+            echo "::warning::Skipping integration test — credentials not configured in 1Password"
+          fi
+
+      - name: Run integration tests
+        if: >-
+          (steps.changes.outputs.cloudflare-metrics == 'true' || github.event_name != 'pull_request')
+          && steps.creds.outputs.available == 'true'
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_TF_DEV_ENV }}
+        run: |
+          op run \
+            --env-file=apps/cloudflare-metrics/.integration.env \
+            -- pnpm --filter @immich-services/cloudflare-metrics run test:integration

.gitignore

@@ -14,6 +14,9 @@ build/
 .wrangler/
 .dev.vars
 
+# Local-only env files (the tracked one is `deployment/.env`)
+deployment/.env.*
+
 # Logs
 logs/
 *.log
@@ -42,3 +45,4 @@ npm-debug.log*
 # Misc
 .cache/
 tmp/
+.claude/

apps/cloudflare-metrics/.integration.env

@@ -0,0 +1,12 @@
+# 1Password-templated env file loaded by `op run` in CI.
+#
+# To run integration tests locally:
+#   op run --env-file=apps/cloudflare-metrics/.integration.env -- \
+#     pnpm --filter @immich-services/cloudflare-metrics run test:integration
+#
+# The 1Password item and field names below are placeholders — set them up
+# in the service account's vault before the CI job can run. Both need to
+# map to a Cloudflare API token that has `Account Analytics:Read` and
+# `Zone:Read` for the dev account.
+CLOUDFLARE_API_TOKEN="op://services-cf-workers-dev/cloudflare-metrics-integration/api_token"
+CLOUDFLARE_ACCOUNT_ID="op://services-cf-workers-dev/cloudflare-metrics-integration/account_id"

apps/cloudflare-metrics/README.md

@@ -0,0 +1,190 @@
+# cloudflare-metrics
+
+Cloudflare Worker that pulls analytics from the Cloudflare GraphQL API every minute, enriches with resource names from the REST API, and writes to VictoriaMetrics as InfluxDB line protocol.
+
+## Architecture
+
+```
+  Cloudflare GraphQL Analytics API ──► CloudflareGraphQLClient
+  Cloudflare REST API (D1/queues/zones) ──► CloudflareRestClient
+                    │
+                    ▼
+           CloudflareMetricsCollector
+            ├─ ResourceCacheService (id → name lookups)
+            ├─ emit.ts (row → Metric translation)
+            └─ graphql-builders.ts (query construction)
+                    │
+                    ▼
+           InfluxMetricsProvider ──► VictoriaMetrics /write
+```
+
+## Collection window
+
+| Setting | Value       | Why                                                           |
+| ------- | ----------- | ------------------------------------------------------------- |
+| Cron    | `* * * * *` | Every minute                                                  |
+| Lag     | 5 min       | Cloudflare's analytics pipeline is 2–5 min behind real time   |
+| Window  | 3 min       | Overlaps consecutive ticks so a missed cron doesn't drop data |
+| Dedup   | Free        | VictoriaMetrics dedupes on `(series, timestamp)`              |
+
+## Datasets
+
+78 datasets across account-scope and zone-scope, covering:
+
+- **Workers**: invocations, subrequests, overview, scheduled (client-side aggregated), analytics engine, builds, VPC, placement, workflows
+- **D1**: queries (summary + detail with p50/p95/p99), storage
+- **R2**: operations, storage, sippy
+- **KV**: operations, storage
+- **Durable Objects**: invocations, periodic, storage, SQL storage, subrequests
+- **Queues**: operations, backlog, consumer concurrency
+- **Hyperdrive**: queries (with count), pool sizes
+- **HTTP (zone-scope)**: overview, detail (per-zone batched), cache reserve, logpush health
+- **Pages Functions**: invocations with CPU/duration p50/p99
+- **AI**: inference, gateway (requests/cache/errors/size), search, autoRAG
+- **Vectorize**: operations, queries, storage, writes
+- **Browser**: rendering API/sessions/time/events, isolation sessions/actions
+- **Stream/Video/Calls**: minutes viewed, CMCD, buffer/playback/quality events, live input, realtime kit, calls usage/TURN
+- **Images/toMarkdown**: request counts, conversion stats
+- **RUM**: pageload, performance (with FCP/page-load p50/p95/p99), web vitals (CLS/FCP/FID/INP/LCP/TTFB averages + p75/p95)
+- **Pipelines**: ingestion, delivery, operator, sink
+- **Containers, Turnstile**
+- **DNS, Email Routing/Sending, DMARC** (zone-scope)
+- **API Gateway sessions, Workers Zone invocations/subrequests** (zone-scope)
+
+See `src/datasets.ts` for the full registry. Each dataset declares its GraphQL field, dimensions, aggregation blocks, tag mappings, and field mappings.
+
+### Skipped datasets
+
+| Dataset                               | Reason                                                                     |
+| ------------------------------------- | -------------------------------------------------------------------------- |
+| `firewallEventsAdaptiveGroups`        | Plan-gated (Business/Enterprise only)                                      |
+| `cdnNetworkAnalyticsAdaptiveGroups`   | Plan-gated                                                                 |
+| `zarazTrack/TriggersAdaptiveGroups`   | Incompatible filter shape (`datetimeMinute_geq` instead of `datetime_geq`) |
+| `cloudchamberMetricsAdaptiveGroups`   | Duplicate schema of `containersMetrics`                                    |
+| `cacheReserveRequestsAdaptiveGroups`  | Plan-gated (zone-scope)                                                    |
+| `healthCheckEventsAdaptiveGroups`     | Plan-gated (zone-scope)                                                    |
+| `loadBalancingRequestsAdaptiveGroups` | Plan-gated (zone-scope)                                                    |
+| `nelReportsAdaptiveGroups`            | Plan-gated (zone-scope)                                                    |
+| `pageShieldReportsAdaptiveGroups`     | Plan-gated (zone-scope)                                                    |
+| `waitingRoomAnalyticsAdaptiveGroups`  | Plan-gated (zone-scope)                                                    |
+
+## Query batching
+
+Cloudflare Workers caps subrequests at 50 per invocation. Account-scope datasets are batched into chunks of 25 using GraphQL aliases. Zone-scope datasets are batched across all zones in a single request per dataset.
+
+| Metric                         | Cold start | Warm (cached)        |
+| ------------------------------ | ---------- | -------------------- |
+| REST lookups (D1/queues/zones) | 3          | 0 (10-min TTL cache) |
+| GraphQL account batches        | 3 chunks   | 3 chunks             |
+| GraphQL date-granularity batch | 1          | 1                    |
+| Zone-scope datasets            | ~11        | ~11                  |
+| Metric flush                   | 1          | 1                    |
+| **Total subrequests**          | **~19**    | **~16**              |
+
+## Resource name enrichment
+
+IDs in the analytics API are enriched with human-readable names via REST lookups:
+
+- `database_name` on `cf_d1_*` metrics (from `/accounts/{id}/d1/database`)
+- `queue_name` on `cf_queue_*` metrics (from `/accounts/{id}/queues`)
+- `zone_name` on `cf_http_*` metrics (from `/zones?account.id={id}` + per-zone fallback for Pages projects)
+
+Module-level caches survive across isolate invocations (typically 10+ minutes on the paid plan). A 10-minute TTL triggers periodic re-fetches. Failed lookups fall back to stale cached names.
+
+## Self-telemetry
+
+The worker emits its own health metrics alongside the Cloudflare data:
+
+- `cloudflare_metrics_cron_summary` — datasets/points/errors per tick
+- `cloudflare_metrics_cron_error{reason}` — early-exit errors
+- `cloudflare_metrics_collector_dataset{dataset,status}` — per-dataset rows/points/duration/errors
+- `cloudflare_metrics_resource_lookup{resource,status}` — REST lookup outcomes
+- `cloudflare_metrics_graphql_client` — requests/error_responses per tick
+- `cloudflare_metrics_flush{status}` — bytes/duration/pending buffers (from previous tick)
+- `cloudflare_metrics_http_response{method,path,status}` — HTTP handler counts
+- `cloudflare_metrics_handle_request` — handler duration/invocation
+
+## Dashboards
+
+20 Grafana dashboards managed via Terraform, in `deployment/.../dashboards/`:
+
+| Dashboard                                      | Template variables              |
+| ---------------------------------------------- | ------------------------------- |
+| Account Overview                               | —                               |
+| Workers                                        | `$script_name`, `$status`       |
+| Workers Scheduled                              | `$script_name`, `$cron`         |
+| D1                                             | `$database_name`                |
+| R2                                             | `$bucket_name`                  |
+| KV                                             | `$namespace_id`                 |
+| Durable Objects                                | `$script_name`, `$namespace_id` |
+| Queues                                         | `$queue_name`                   |
+| Hyperdrive                                     | `$config_id`                    |
+| HTTP / Zones                                   | `$zone_name`                    |
+| Pages Functions                                | `$script_name`                  |
+| AI, Vectorize, Browser, Stream, RUM, Pipelines | —                               |
+| DNS                                            | `$zone_name`                    |
+| Email                                          | `$zone_name`                    |
+| Exporter Health                                | —                               |
+
+## Alerts
+
+7 Grafana alert rules in `deployment/.../alerts.tf`, all linked to the exporter-health dashboard:
+
+| Rule                               | Condition                          | Severity |
+| ---------------------------------- | ---------------------------------- | -------- |
+| Collector Not Running              | No `cron_summary_datasets` for 10m | 1        |
+| Collector Cron Error               | `cron_error_count > 0` in 10m      | 1        |
+| Dataset Errors Sustained           | `>10` errors in 15m                | 3        |
+| Metrics Flush Failing              | `>3` flush errors in 15m           | 1        |
+| Pending Flush Buffer Growing       | `>3` stashed bodies for 10m        | 3        |
+| GraphQL Subrequest Budget Near Cap | `>40` requests/tick for 10m        | 3        |
+| GraphQL Error Responses Sustained  | `>5` error responses in 15m        | 3        |
+
+## File structure
+
+```
+src/
+  index.ts                    Entry point (wires handlers)
+  handlers/
+    http.ts                   /health endpoint
+    scheduled.ts              Cron handler (collect + flush)
+  collector.ts                Orchestration (collectAll → batched account/zone fetches)
+  resource-cache.ts           REST resource lookups + module-level caching
+  emit.ts                     DatasetRow → Metric translation + tag enrichment
+  graphql-client.ts           CloudflareGraphQLClient (HTTP transport + chunking)
+  graphql-builders.ts         Query construction (pure functions)
+  cloudflare-api.ts           CloudflareRestClient (D1/queues/zones REST)
+  metrics.ts                  CloudflareMetricsRepository facade
+  metric.ts                   Metric data class
+  metric-providers.ts         InfluxMetricsProvider + HeaderMetricsProvider
+  flush-state.ts              Retry buffer + last-flush stats (module-level state)
+  datasets.ts                 78 DatasetQuery definitions
+  types.ts                    Shared types
+  deferred.ts                 DeferredRepository (waitUntil helper)
+  monitor.ts                  monitorAsyncFunction (duration/invocation wrapper)
+```
+
+## Development
+
+```bash
+pnpm run dev               # wrangler dev (local)
+pnpm run test              # 71 unit tests
+pnpm run test:integration  # live API tests (needs CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID)
+pnpm run check             # tsc --noEmit
+pnpm run build             # wrangler deploy --dry-run
+```
+
+Local `.dev.vars`:
+
+```
+CLOUDFLARE_API_TOKEN=...
+CLOUDFLARE_ACCOUNT_ID=...
+VMETRICS_API_TOKEN=...
+ENVIRONMENT=dev
+```
+
+## Infrastructure
+
+- **Worker**: `apps/cloudflare-metrics/` — TypeScript, Wrangler, every-minute cron
+- **Terraform**: `deployment/modules/cloudflare/workers/cloudflare-metrics/` — worker, version, deployment, cron trigger, dashboards, alerts, and a scoped API token with Account Analytics Read + D1 Read + Queues Read + Zone Read
+- **CI**: unit tests on every PR, path-filtered integration tests (gated on 1Password credentials), build + deploy-dev on PR, deploy-prod on merge to main

apps/cloudflare-metrics/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@immich-services/cloudflare-metrics",
+  "version": "1.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "wrangler dev",
+    "build": "wrangler deploy --dry-run --outdir ../../dist/cloudflare-metrics",
+    "tail": "wrangler tail",
+    "test": "vitest run --exclude 'src/integration.test.ts'",
+    "test:integration": "vitest run --config vitest.integration.config.ts",
+    "check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@influxdata/influxdb-client": "^1.34.0"
+  }
+}