Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions charts/glassflow-etl/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.5.17
version: 0.5.18

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
Expand All @@ -41,7 +41,14 @@ dependencies:
version: "0.1.0"
repository: https://glassflow.github.io/charts
condition: global.pgbouncer.enabled
alias: pgbouncer
- name: victoria-metrics-single
version: "0.37.0"
repository: https://victoriametrics.github.io/helm-charts/
condition: global.observability.metrics.internal.enabled
- name: victoria-logs-single
version: "0.12.4"
repository: https://victoriametrics.github.io/helm-charts/
condition: global.observability.logs.internal.enabled

maintainers:
- name: Glassflow
Expand Down
19 changes: 19 additions & 0 deletions charts/glassflow-etl/templates/NOTES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,22 @@
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
{{- if or .Values.global.observability.metrics.internal.enabled .Values.global.observability.logs.internal.enabled }}

2. Internal observability is enabled. The OTel collector fans out to the bundled
destinations in addition to any external exporters.
{{- if .Values.global.observability.metrics.internal.enabled }}

VictoriaMetrics: http://{{ include "glassflow-etl.internalVMService" . }}.{{ .Release.Namespace }}.svc.cluster.local:8428
kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "glassflow-etl.internalVMService" . }} 8428
curl 'http://localhost:8428/api/v1/query?query=gfm_ingestor_consumed_total'
{{- end }}
{{- if .Values.global.observability.logs.internal.enabled }}

VictoriaLogs: http://{{ include "glassflow-etl.internalVLService" . }}.{{ .Release.Namespace }}.svc.cluster.local:9428
kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "glassflow-etl.internalVLService" . }} 9428
curl 'http://localhost:9428/select/logsql/query?query=service.name:glassflow-api'
{{- end }}

GlassFlow API query endpoints arrive in M2 (ETL-1014).
{{- end }}
30 changes: 30 additions & 0 deletions charts/glassflow-etl/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,33 @@ Create the name of the usage stats service account to use
{{- define "glassflow-etl.usageStatsServiceAccountName" -}}
{{- include "glassflow-etl.fullname" . }}-usage-stats
{{- end }}

{{/*
Validate observability config. Fails the install with a clear error when an
internal subchart is requested without its parent signal enabled.
Invoked unconditionally from otel-collector-configmap.yaml (and any other
template) so misconfigurations are caught even when the surrounding template
itself doesn't render.
*/}}
{{- define "glassflow-etl.observability.validate" -}}
{{- if and .Values.global.observability.metrics.internal.enabled (not .Values.global.observability.metrics.enabled) -}}
{{- fail "global.observability.metrics.internal.enabled requires global.observability.metrics.enabled=true" -}}
{{- end -}}
{{- if and .Values.global.observability.logs.internal.enabled (not .Values.global.observability.logs.enabled) -}}
{{- fail "global.observability.logs.internal.enabled requires global.observability.logs.enabled=true" -}}
{{- end -}}
{{- end }}

{{/*
DNS name of the bundled VictoriaMetrics service (subchart-managed).
*/}}
{{- define "glassflow-etl.internalVMService" -}}
{{ .Release.Name }}-victoria-metrics-single-server
{{- end }}

{{/*
DNS name of the bundled VictoriaLogs service (subchart-managed).
*/}}
{{- define "glassflow-etl.internalVLService" -}}
{{ .Release.Name }}-victoria-logs-single-server
{{- end }}
51 changes: 38 additions & 13 deletions charts/glassflow-etl/templates/otel-collector-configmap.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
{{- if or .Values.global.observability.metrics.enabled .Values.global.observability.logs.enabled }}
{{- /* Validate observability config first — fails install on bad combinations. */ -}}
{{- include "glassflow-etl.observability.validate" . -}}
{{- $metricsEnabled := .Values.global.observability.metrics.enabled -}}
{{- $logsEnabled := .Values.global.observability.logs.enabled -}}
{{- $metricsInternal := .Values.global.observability.metrics.internal.enabled -}}
{{- $logsInternal := .Values.global.observability.logs.internal.enabled -}}
{{- if or $metricsEnabled $logsEnabled }}
apiVersion: v1
kind: ConfigMap
metadata:
Expand All @@ -13,7 +19,7 @@ data:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
{{- if and .Values.nats.enabled .Values.global.observability.metrics.enabled }}
{{- if and .Values.nats.enabled $metricsEnabled }}
prometheus:
config:
scrape_configs:
Expand All @@ -39,24 +45,30 @@ data:
- key: service.namespace
from_attribute: service.namespace
action: insert
{{- if .Values.global.observability.logs.enabled }}
{{- if $logsEnabled }}
{{- range $name, $cfg := (default dict .Values.global.observability.logs.extraProcessors) }}
{{ $name }}: {{- toYaml $cfg | nindent 8 }}
{{- end }}
{{- end }}
{{- if .Values.global.observability.metrics.enabled }}
{{- if $metricsEnabled }}
{{- range $name, $cfg := (default dict .Values.global.observability.metrics.extraProcessors) }}
{{ $name }}: {{- toYaml $cfg | nindent 8 }}
{{- end }}
{{- end }}

exporters:
{{- if .Values.global.observability.logs.enabled }}
{{- if $logsEnabled }}
{{- range $name, $cfg := .Values.global.observability.logs.exporter }}
{{ $name }}: {{- toYaml $cfg | nindent 8 }}
{{- end }}
{{- if $logsInternal }}
otlphttp/internal_logs:
endpoint: "http://{{ include "glassflow-etl.internalVLService" . }}:9428/insert/opentelemetry"
tls:
insecure: true
{{- end }}
{{- end }}
{{- if .Values.global.observability.metrics.enabled }}
{{- if $metricsEnabled }}
prometheus:
endpoint: "0.0.0.0:9090"
namespace: {{ .Release.Namespace }}
Expand All @@ -65,32 +77,45 @@ data:
{{- range $name, $cfg := (default dict .Values.global.observability.metrics.extraExporters) }}
{{ $name }}: {{- toYaml $cfg | nindent 8 }}
{{- end }}
{{- if $metricsInternal }}
prometheusremotewrite/internal:
endpoint: "http://{{ include "glassflow-etl.internalVMService" . }}:8428/api/v1/write"
tls:
insecure: true
{{- end }}
{{- end }}

extensions:
health_check:
endpoint: 0.0.0.0:13133

service:
extensions: [health_check]
pipelines:
{{- if .Values.global.observability.logs.enabled }}
{{- $logExporters := keys .Values.global.observability.logs.exporter | sortAlpha -}}
{{- if $logsEnabled }}
{{- $logExporters := keys (default dict .Values.global.observability.logs.exporter) | sortAlpha }}
{{- if $logsInternal }}
{{- $logExporters = append $logExporters "otlphttp/internal_logs" }}
{{- end }}
{{- if eq (len $logExporters) 0 -}}
{{- fail "logs.enabled=true requires at least one exporter configured under global.observability.logs.exporter" -}}
{{- fail "logs.enabled=true requires at least one exporter — configure global.observability.logs.exporter or set global.observability.logs.internal.enabled=true" -}}
{{- end }}
{{- $logExtraProcessors := keys (default dict .Values.global.observability.logs.extraProcessors) | sortAlpha }}
logs:
receivers: [otlp]
processors: [{{ if $logExtraProcessors }}{{ join ", " $logExtraProcessors }}, {{ end }}resource, batch]
exporters: [{{ join ", " $logExporters }}]
{{- end }}
{{- if .Values.global.observability.metrics.enabled }}
{{- $metricsExtraExporters := keys (default dict .Values.global.observability.metrics.extraExporters) | sortAlpha }}
{{- if $metricsEnabled }}
{{- $metricsExporters := keys (default dict .Values.global.observability.metrics.extraExporters) | sortAlpha }}
{{- $metricsExporters = prepend $metricsExporters "prometheus" }}
{{- if $metricsInternal }}
{{- $metricsExporters = append $metricsExporters "prometheusremotewrite/internal" }}
{{- end }}
{{- $metricsExtraProcessors := keys (default dict .Values.global.observability.metrics.extraProcessors) | sortAlpha }}
metrics:
receivers: [otlp{{- if .Values.nats.enabled }}, prometheus{{- end }}]
processors: [{{ if $metricsExtraProcessors }}{{ join ", " $metricsExtraProcessors }}, {{ end }}resource, batch]
exporters: [prometheus{{ if $metricsExtraExporters }}, {{ join ", " $metricsExtraExporters }}{{ end }}]
exporters: [{{ join ", " $metricsExporters }}]
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{{- $vm := .Values.global.observability.metrics.internal.enabled -}}
{{- $vl := .Values.global.observability.logs.internal.enabled -}}
{{- if or $vm $vl }}
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "glassflow-etl.fullname" . }}-test-internal-observability"
labels:
{{- include "glassflow-etl.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
spec:
restartPolicy: Never
containers:
{{- if $vm }}
- name: probe-vmsingle
image: busybox
command: ['wget']
args:
- '-qO-'
- 'http://{{ include "glassflow-etl.internalVMService" . }}:8428/health'
{{- end }}
{{- if $vl }}
- name: probe-vlogs
image: busybox
command: ['wget']
args:
- '-qO-'
- 'http://{{ include "glassflow-etl.internalVLService" . }}:9428/health'
{{- end }}
{{- end }}
107 changes: 107 additions & 0 deletions charts/glassflow-etl/values.dev.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# Dev / local-cluster overrides for charts/glassflow-etl.
#
# Cuts resource requests + replicas to fit on a laptop kind / docker-desktop
# cluster. Not for staging or production.
#
# Usage:
# helm install glassflow ./packages/glassflow-etl-0.5.18.tgz \
# -n glassflow --create-namespace \
# -f charts/glassflow-etl/values.dev.yaml

# -----------------------------------------------------------------------------
# Internal observability ON for dev — exercises the bundled VM + VL stack.
# Note: VM/VL persistence + resources are driven by the upstream subcharts'
# top-level blocks below (victoria-metrics-single / victoria-logs-single).
# The internal.enabled flags here only gate "subchart deployed + collector
# fan-out wired"; they require the parent metrics/logs.enabled to be true.
# -----------------------------------------------------------------------------
global:
observability:
metrics:
enabled: true
internal:
enabled: true
logs:
# Flip to true so the operator stamps GLASSFLOW_OTEL_LOGS_ENABLED=true on
# data-plane Pods (ingestor/sink/dedup/join), otherwise their logs never
# reach VictoriaLogs.
enabled: true
internal:
enabled: true
Comment thread
PabloPardoGarcia marked this conversation as resolved.
Resolved
otelCollector:
resources:
requests: { cpu: 50m, memory: 64Mi }
limits: { cpu: 200m, memory: 256Mi }

victoria-metrics-single:
server:
retentionPeriod: "1d"
persistentVolume:
enabled: true
size: 1Gi
resources:
requests: { cpu: 50m, memory: 128Mi }
limits: { cpu: 200m, memory: 512Mi }

victoria-logs-single:
server:
retentionPeriod: "1d"
persistentVolume:
enabled: true
size: 1Gi
resources:
requests: { cpu: 25m, memory: 64Mi }
limits: { cpu: 100m, memory: 256Mi }

# -----------------------------------------------------------------------------
# NATS — single replica, 1 CPU, 1 GiB RAM, 5 GiB disk.
# Default chart ships 3×4cpu/3Gi/100Gi which won't fit on a laptop.
# -----------------------------------------------------------------------------
nats:
config:
cluster:
replicas: 3
jetstream:
fileStore:
pvc:
size: 5Gi
container:
merge:
resources:
requests: { cpu: "1000m", memory: "1Gi" }
limits: { cpu: "1000m", memory: "1Gi" }

# -----------------------------------------------------------------------------
# PostgreSQL — single replica, dev-only password (don't use these in prod).
# -----------------------------------------------------------------------------
postgresql:
replicaCount: 1
persistence:
size: 2Gi

# -----------------------------------------------------------------------------
# GlassFlow components — modest requests, generous limits for room to grow.
# -----------------------------------------------------------------------------
api:
resources:
requests: { cpu: 50m, memory: 128Mi }
limits: { cpu: 500m, memory: 512Mi }

ui:
resources:
requests: { cpu: 50m, memory: 256Mi }
limits: { cpu: 500m, memory: 512Mi }

# Operator chart (subchart values nested under its alias).
glassflow-operator:
controllerManager:
manager:
resources:
requests: { cpu: 50m, memory: 64Mi }
limits: { cpu: 500m, memory: 256Mi }

# Notification service — only meaningful if global.notifications.enabled=true.
notificationService:
resources:
requests: { cpu: 25m, memory: 64Mi }
limits: { cpu: 200m, memory: 128Mi }
Loading
Loading