From a4edd3ad00f1c1d37551effbb96bf0e625ec1c01 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Fri, 8 May 2026 13:49:27 +0200 Subject: [PATCH] feat(chart): add internal observability with VictoriaMetrics + VictoriaLogs (ETL-1073) --- charts/glassflow-etl/Chart.yaml | 11 +- charts/glassflow-etl/templates/NOTES.txt | 19 +++ charts/glassflow-etl/templates/_helpers.tpl | 30 ++++ .../templates/otel-collector-configmap.yaml | 51 +++++-- .../tests/test-internal-observability.yaml | 32 +++++ charts/glassflow-etl/values.dev.yaml | 107 ++++++++++++++ charts/glassflow-etl/values.yaml | 135 ++++++++++++++---- ct.yaml | 1 + 8 files changed, 343 insertions(+), 43 deletions(-) create mode 100644 charts/glassflow-etl/templates/tests/test-internal-observability.yaml create mode 100644 charts/glassflow-etl/values.dev.yaml diff --git a/charts/glassflow-etl/Chart.yaml b/charts/glassflow-etl/Chart.yaml index 2efc357..bcb0557 100644 --- a/charts/glassflow-etl/Chart.yaml +++ b/charts/glassflow-etl/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.17 +version: 0.5.18 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -41,7 +41,14 @@ dependencies: version: "0.1.0" repository: https://glassflow.github.io/charts condition: global.pgbouncer.enabled - alias: pgbouncer + - name: victoria-metrics-single + version: "0.37.0" + repository: https://victoriametrics.github.io/helm-charts/ + condition: global.observability.metrics.internal.enabled + - name: victoria-logs-single + version: "0.12.4" + repository: https://victoriametrics.github.io/helm-charts/ + condition: global.observability.logs.internal.enabled maintainers: - name: Glassflow diff --git a/charts/glassflow-etl/templates/NOTES.txt b/charts/glassflow-etl/templates/NOTES.txt index 60eb1b1..c5dd65a 100644 --- a/charts/glassflow-etl/templates/NOTES.txt +++ b/charts/glassflow-etl/templates/NOTES.txt @@ -20,3 +20,22 @@ echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT {{- end }} +{{- if or .Values.global.observability.metrics.internal.enabled .Values.global.observability.logs.internal.enabled }} + +2. Internal observability is enabled. The OTel collector fans out to the bundled + destinations in addition to any external exporters. +{{- if .Values.global.observability.metrics.internal.enabled }} + + VictoriaMetrics: http://{{ include "glassflow-etl.internalVMService" . }}.{{ .Release.Namespace }}.svc.cluster.local:8428 + kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "glassflow-etl.internalVMService" . }} 8428 + curl 'http://localhost:8428/api/v1/query?query=gfm_ingestor_consumed_total' +{{- end }} +{{- if .Values.global.observability.logs.internal.enabled }} + + VictoriaLogs: http://{{ include "glassflow-etl.internalVLService" . }}.{{ .Release.Namespace }}.svc.cluster.local:9428 + kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "glassflow-etl.internalVLService" . }} 9428 + curl 'http://localhost:9428/select/logsql/query?query=service.name:glassflow-api' +{{- end }} + + GlassFlow API query endpoints arrive in M2 (ETL-1014). +{{- end }} diff --git a/charts/glassflow-etl/templates/_helpers.tpl b/charts/glassflow-etl/templates/_helpers.tpl index 998fa7a..4f61220 100644 --- a/charts/glassflow-etl/templates/_helpers.tpl +++ b/charts/glassflow-etl/templates/_helpers.tpl @@ -67,3 +67,33 @@ Create the name of the usage stats service account to use {{- define "glassflow-etl.usageStatsServiceAccountName" -}} {{- include "glassflow-etl.fullname" . }}-usage-stats {{- end }} + +{{/* +Validate observability config. Fails the install with a clear error when an +internal subchart is requested without its parent signal enabled. +Invoked unconditionally from otel-collector-configmap.yaml (and any other +template) so misconfigurations are caught even when the surrounding template +itself doesn't render. +*/}} +{{- define "glassflow-etl.observability.validate" -}} +{{- if and .Values.global.observability.metrics.internal.enabled (not .Values.global.observability.metrics.enabled) -}} +{{- fail "global.observability.metrics.internal.enabled requires global.observability.metrics.enabled=true" -}} +{{- end -}} +{{- if and .Values.global.observability.logs.internal.enabled (not .Values.global.observability.logs.enabled) -}} +{{- fail "global.observability.logs.internal.enabled requires global.observability.logs.enabled=true" -}} +{{- end -}} +{{- end }} + +{{/* +DNS name of the bundled VictoriaMetrics service (subchart-managed). +*/}} +{{- define "glassflow-etl.internalVMService" -}} +{{ .Release.Name }}-victoria-metrics-single-server +{{- end }} + +{{/* +DNS name of the bundled VictoriaLogs service (subchart-managed). +*/}} +{{- define "glassflow-etl.internalVLService" -}} +{{ .Release.Name }}-victoria-logs-single-server +{{- end }} diff --git a/charts/glassflow-etl/templates/otel-collector-configmap.yaml b/charts/glassflow-etl/templates/otel-collector-configmap.yaml index e7de8f3..9781fcb 100644 --- a/charts/glassflow-etl/templates/otel-collector-configmap.yaml +++ b/charts/glassflow-etl/templates/otel-collector-configmap.yaml @@ -1,4 +1,10 @@ -{{- if or .Values.global.observability.metrics.enabled .Values.global.observability.logs.enabled }} +{{- /* Validate observability config first — fails install on bad combinations. */ -}} +{{- include "glassflow-etl.observability.validate" . -}} +{{- $metricsEnabled := .Values.global.observability.metrics.enabled -}} +{{- $logsEnabled := .Values.global.observability.logs.enabled -}} +{{- $metricsInternal := .Values.global.observability.metrics.internal.enabled -}} +{{- $logsInternal := .Values.global.observability.logs.internal.enabled -}} +{{- if or $metricsEnabled $logsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -13,7 +19,7 @@ data: endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 -{{- if and .Values.nats.enabled .Values.global.observability.metrics.enabled }} +{{- if and .Values.nats.enabled $metricsEnabled }} prometheus: config: scrape_configs: @@ -39,24 +45,30 @@ data: - key: service.namespace from_attribute: service.namespace action: insert -{{- if .Values.global.observability.logs.enabled }} +{{- if $logsEnabled }} {{- range $name, $cfg := (default dict .Values.global.observability.logs.extraProcessors) }} {{ $name }}: {{- toYaml $cfg | nindent 8 }} {{- end }} {{- end }} -{{- if .Values.global.observability.metrics.enabled }} +{{- if $metricsEnabled }} {{- range $name, $cfg := (default dict .Values.global.observability.metrics.extraProcessors) }} {{ $name }}: {{- toYaml $cfg | nindent 8 }} {{- end }} {{- end }} exporters: -{{- if .Values.global.observability.logs.enabled }} +{{- if $logsEnabled }} {{- range $name, $cfg := .Values.global.observability.logs.exporter }} {{ $name }}: {{- toYaml $cfg | nindent 8 }} {{- end }} + {{- if $logsInternal }} + otlphttp/internal_logs: + endpoint: "http://{{ include "glassflow-etl.internalVLService" . }}:9428/insert/opentelemetry" + tls: + insecure: true + {{- end }} {{- end }} -{{- if .Values.global.observability.metrics.enabled }} +{{- if $metricsEnabled }} prometheus: endpoint: "0.0.0.0:9090" namespace: {{ .Release.Namespace }} @@ -65,8 +77,14 @@ data: {{- range $name, $cfg := (default dict .Values.global.observability.metrics.extraExporters) }} {{ $name }}: {{- toYaml $cfg | nindent 8 }} {{- end }} + {{- if $metricsInternal }} + prometheusremotewrite/internal: + endpoint: "http://{{ include "glassflow-etl.internalVMService" . }}:8428/api/v1/write" + tls: + insecure: true + {{- end }} {{- end }} - + extensions: health_check: endpoint: 0.0.0.0:13133 @@ -74,10 +92,13 @@ data: service: extensions: [health_check] pipelines: -{{- if .Values.global.observability.logs.enabled }} - {{- $logExporters := keys .Values.global.observability.logs.exporter | sortAlpha -}} +{{- if $logsEnabled }} + {{- $logExporters := keys (default dict .Values.global.observability.logs.exporter) | sortAlpha }} + {{- if $logsInternal }} + {{- $logExporters = append $logExporters "otlphttp/internal_logs" }} + {{- end }} {{- if eq (len $logExporters) 0 -}} - {{- fail "logs.enabled=true requires at least one exporter configured under global.observability.logs.exporter" -}} + {{- fail "logs.enabled=true requires at least one exporter — configure global.observability.logs.exporter or set global.observability.logs.internal.enabled=true" -}} {{- end }} {{- $logExtraProcessors := keys (default dict .Values.global.observability.logs.extraProcessors) | sortAlpha }} logs: @@ -85,12 +106,16 @@ data: processors: [{{ if $logExtraProcessors }}{{ join ", " $logExtraProcessors }}, {{ end }}resource, batch] exporters: [{{ join ", " $logExporters }}] {{- end }} -{{- if .Values.global.observability.metrics.enabled }} - {{- $metricsExtraExporters := keys (default dict .Values.global.observability.metrics.extraExporters) | sortAlpha }} +{{- if $metricsEnabled }} + {{- $metricsExporters := keys (default dict .Values.global.observability.metrics.extraExporters) | sortAlpha }} + {{- $metricsExporters = prepend $metricsExporters "prometheus" }} + {{- if $metricsInternal }} + {{- $metricsExporters = append $metricsExporters "prometheusremotewrite/internal" }} + {{- end }} {{- $metricsExtraProcessors := keys (default dict .Values.global.observability.metrics.extraProcessors) | sortAlpha }} metrics: receivers: [otlp{{- if .Values.nats.enabled }}, prometheus{{- end }}] processors: [{{ if $metricsExtraProcessors }}{{ join ", " $metricsExtraProcessors }}, {{ end }}resource, batch] - exporters: [prometheus{{ if $metricsExtraExporters }}, {{ join ", " $metricsExtraExporters }}{{ end }}] + exporters: [{{ join ", " $metricsExporters }}] {{- end }} {{- end }} diff --git a/charts/glassflow-etl/templates/tests/test-internal-observability.yaml b/charts/glassflow-etl/templates/tests/test-internal-observability.yaml new file mode 100644 index 0000000..0a682e1 --- /dev/null +++ b/charts/glassflow-etl/templates/tests/test-internal-observability.yaml @@ -0,0 +1,32 @@ +{{- $vm := .Values.global.observability.metrics.internal.enabled -}} +{{- $vl := .Values.global.observability.logs.internal.enabled -}} +{{- if or $vm $vl }} +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "glassflow-etl.fullname" . }}-test-internal-observability" + labels: + {{- include "glassflow-etl.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: +{{- if $vm }} + - name: probe-vmsingle + image: busybox + command: ['wget'] + args: + - '-qO-' + - 'http://{{ include "glassflow-etl.internalVMService" . }}:8428/health' +{{- end }} +{{- if $vl }} + - name: probe-vlogs + image: busybox + command: ['wget'] + args: + - '-qO-' + - 'http://{{ include "glassflow-etl.internalVLService" . }}:9428/health' +{{- end }} +{{- end }} diff --git a/charts/glassflow-etl/values.dev.yaml b/charts/glassflow-etl/values.dev.yaml new file mode 100644 index 0000000..c32a7fb --- /dev/null +++ b/charts/glassflow-etl/values.dev.yaml @@ -0,0 +1,107 @@ +# Dev / local-cluster overrides for charts/glassflow-etl. +# +# Cuts resource requests + replicas to fit on a laptop kind / docker-desktop +# cluster. Not for staging or production. +# +# Usage: +# helm install glassflow ./packages/glassflow-etl-0.5.18.tgz \ +# -n glassflow --create-namespace \ +# -f charts/glassflow-etl/values.dev.yaml + +# ----------------------------------------------------------------------------- +# Internal observability ON for dev — exercises the bundled VM + VL stack. +# Note: VM/VL persistence + resources are driven by the upstream subcharts' +# top-level blocks below (victoria-metrics-single / victoria-logs-single). +# The internal.enabled flags here only gate "subchart deployed + collector +# fan-out wired"; they require the parent metrics/logs.enabled to be true. +# ----------------------------------------------------------------------------- +global: + observability: + metrics: + enabled: true + internal: + enabled: true + logs: + # Flip to true so the operator stamps GLASSFLOW_OTEL_LOGS_ENABLED=true on + # data-plane Pods (ingestor/sink/dedup/join), otherwise their logs never + # reach VictoriaLogs. + enabled: true + internal: + enabled: true + otelCollector: + resources: + requests: { cpu: 50m, memory: 64Mi } + limits: { cpu: 200m, memory: 256Mi } + +victoria-metrics-single: + server: + retentionPeriod: "1d" + persistentVolume: + enabled: true + size: 1Gi + resources: + requests: { cpu: 50m, memory: 128Mi } + limits: { cpu: 200m, memory: 512Mi } + +victoria-logs-single: + server: + retentionPeriod: "1d" + persistentVolume: + enabled: true + size: 1Gi + resources: + requests: { cpu: 25m, memory: 64Mi } + limits: { cpu: 100m, memory: 256Mi } + +# ----------------------------------------------------------------------------- +# NATS — single replica, 1 CPU, 1 GiB RAM, 5 GiB disk. +# Default chart ships 3×4cpu/3Gi/100Gi which won't fit on a laptop. +# ----------------------------------------------------------------------------- +nats: + config: + cluster: + replicas: 3 + jetstream: + fileStore: + pvc: + size: 5Gi + container: + merge: + resources: + requests: { cpu: "1000m", memory: "1Gi" } + limits: { cpu: "1000m", memory: "1Gi" } + +# ----------------------------------------------------------------------------- +# PostgreSQL — single replica, dev-only password (don't use these in prod). +# ----------------------------------------------------------------------------- +postgresql: + replicaCount: 1 + persistence: + size: 2Gi + +# ----------------------------------------------------------------------------- +# GlassFlow components — modest requests, generous limits for room to grow. +# ----------------------------------------------------------------------------- +api: + resources: + requests: { cpu: 50m, memory: 128Mi } + limits: { cpu: 500m, memory: 512Mi } + +ui: + resources: + requests: { cpu: 50m, memory: 256Mi } + limits: { cpu: 500m, memory: 512Mi } + +# Operator chart (subchart values nested under its alias). +glassflow-operator: + controllerManager: + manager: + resources: + requests: { cpu: 50m, memory: 64Mi } + limits: { cpu: 500m, memory: 256Mi } + +# Notification service — only meaningful if global.notifications.enabled=true. +notificationService: + resources: + requests: { cpu: 25m, memory: 64Mi } + limits: { cpu: 200m, memory: 128Mi } diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index 32fff45..70a22db 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -9,13 +9,35 @@ global: # Global image registry - if set, will be prepended to all image repositories imageRegistry: "ghcr.io/glassflow/" + + # ============================================================================= + # OBSERVABILITY + # ============================================================================= + # Each signal (metrics, logs) has its own block. Inside each: + # - `enabled` — gates the entire signal: the OTel collector pipeline, + # component-side emission, and any internal/external + # destination underneath. Disabling the signal disables + # everything below it. + # - `extraExporters` / `exporter` — external destinations (user-managed backends + # like Prometheus, Loki, Datadog). Multiple destinations + # fan out. + # - `internal.enabled` — bundled destination for that signal: + # metrics → VictoriaMetrics + # logs → VictoriaLogs + # Has no effect unless the parent `enabled: true`. + # Setting `internal.enabled=true` alongside `enabled=false` + # fails the install with a clear error. + # + # Detailed knobs for the bundled VM/VL subcharts (retention, storage, resources) + # live in the top-level `victoria-metrics-single:` / `victoria-logs-single:` + # blocks at the bottom of this file — Helm requires subchart values at the top + # level, not nested under the parent's keys. observability: metrics: enabled: true - # Additional exporters added alongside the built-in Prometheus exporter. - # The Prometheus scrape endpoint (port 9090) is always enabled when metrics.enabled=true. - # Use this map to push metrics to additional backends (e.g. otlp, otlphttp). - # To disable metrics export entirely, set metrics.enabled: false. + # External destinations. The built-in Prometheus :9090 scrape endpoint is + # always exposed when metrics.enabled=true. Use this map to push metrics to + # additional backends (e.g. otlp, otlphttp). # Example: # extraExporters: # otlp: @@ -24,51 +46,46 @@ global: # insecure: true extraExporters: {} # Extra processors added to the metrics pipeline before the default resource and batch processors. - # Keys are OTel processor names, values are their config. - # Use the name/label convention (e.g. memory_limiter/metrics) for multiple instances of the same - # type, and to avoid collisions when both metrics and logs extraProcessors are enabled — both - # blocks are rendered into the same global OTel processors section, so a shared key (e.g. - # memory_limiter) with different configs produces a duplicate YAML key and the last value wins - # silently. Unique names per signal prevent this. + # Use signal-scoped names (e.g. memory_limiter/metrics) to avoid collisions + # when both signals define a same-named processor — both are rendered into + # the shared OTel processors section and the last one wins silently. # Example: # extraProcessors: # memory_limiter/metrics: # check_interval: 1s # limit_mib: 512 - # resource/metrics: - # attributes: - # - key: deployment.environment - # value: production - # action: insert extraProcessors: {} + # Bundled internal destination: VictoriaMetrics. Only takes effect when + # metrics.enabled=true above. Tune retention/storage/resources via the + # top-level `victoria-metrics-single:` block. + internal: + enabled: false + logs: enabled: false - # Exporters map: keys are OTel exporter names, values are their config. - # At least one exporter must be configured when logs.enabled=true. - # Any OTel-supported exporter type is accepted (otlp, otlphttp, etc.). + # External destinations. At least one exporter must be configured when + # logs.enabled=true and internal.enabled=false. # Example: # exporter: - # otlp: - # endpoint: "my-collector:4317" - # tls: - # insecure: true # otlphttp: # endpoint: "http://my-collector:4318" # headers: # Authorization: "Bearer " exporter: {} - # Extra processors added to the logs pipeline before the default resource and batch processors. - # Keys are OTel processor names, values are their config. - # WARNING: if both metrics and logs extraProcessors define the same key with different configs, - # both are rendered into the shared OTel processors section — the last definition wins silently. - # Use signal-scoped names (e.g. memory_limiter/logs) to avoid this. + # Extra processors. Same name-collision warning as metrics.extraProcessors. # Example: # extraProcessors: # memory_limiter/logs: # check_interval: 1s # limit_mib: 256 extraProcessors: {} - # OpenTelemetry collector resources (used when metrics or logs are enabled) + # Bundled internal destination: VictoriaLogs. Only takes effect when + # logs.enabled=true above. Tune retention/storage/resources via the + # top-level `victoria-logs-single:` block. + internal: + enabled: false + + # OpenTelemetry collector resources (used when any signal is enabled). otelCollector: resources: requests: @@ -77,6 +94,7 @@ global: limits: cpu: 500m memory: 512Mi + notifications: # configure slack / email channel details in notificationService section below enabled: true nats: @@ -765,3 +783,64 @@ notificationService: # port: 8080 # initialDelaySeconds: 5 # periodSeconds: 5 + +# ============================================================================= +# VICTORIAMETRICS (BUNDLED) — STORAGE / RETENTION / RESOURCES +# ============================================================================= +# Passed verbatim to the upstream `victoria-metrics-single` subchart. +# Helm requires subchart values at the top level (not nested under the parent's +# observability block), so this is where you tune retention, persistent storage, +# pod resources, ingress, TLS, auth, and any other upstream chart knob. +# +# Only takes effect when global.observability.metrics.internal.enabled=true. +# +# Service exposed at: -victoria-metrics-single-server..svc.cluster.local:8428 +# +# Full upstream values reference: +# https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-single +victoria-metrics-single: + server: + enabled: true + retentionPeriod: "7d" + persistentVolume: + enabled: true + size: 10Gi + # storageClass: "" # uncomment to pin a StorageClass + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + service: + servicePort: 8428 + +# ============================================================================= +# VICTORIALOGS (BUNDLED) — STORAGE / RETENTION / RESOURCES +# ============================================================================= +# Passed verbatim to the upstream `victoria-logs-single` subchart. Same +# reasoning as VictoriaMetrics above — Helm subchart values must live at the +# top level, so retention / storage / resources are tuned here. +# +# Only takes effect when global.observability.logs.internal.enabled=true. +# +# Service exposed at: -victoria-logs-single-server..svc.cluster.local:9428 +# +# Full upstream values reference: +# https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-logs-single +victoria-logs-single: + server: + enabled: true + retentionPeriod: "3d" + persistentVolume: + enabled: true + size: 5Gi + # storageClass: "" + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 512Mi diff --git a/ct.yaml b/ct.yaml index 7ca21fb..48095a3 100644 --- a/ct.yaml +++ b/ct.yaml @@ -10,4 +10,5 @@ chart-repos: - nats=https://nats-io.github.io/k8s/helm/charts/ - glassflow-operator=https://glassflow.github.io/glassflow-etl-k8s-operator - glassflow-charts=https://glassflow.github.io/charts + - vm=https://victoriametrics.github.io/helm-charts/