From 64a721205851a5ff167804947082c3249a56f3a1 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Wed, 1 Apr 2026 13:37:07 +0200 Subject: [PATCH 01/13] Add otlp receiver deployment --- .../templates/otlp-receiver-configmap.yaml | 16 ++ .../templates/otlp-receiver-deployment.yaml | 159 ++++++++++++++++++ .../templates/otlp-receiver-service.yaml | 24 +++ 3 files changed, 199 insertions(+) create mode 100644 charts/glassflow-etl/templates/otlp-receiver-configmap.yaml create mode 100644 charts/glassflow-etl/templates/otlp-receiver-deployment.yaml create mode 100644 charts/glassflow-etl/templates/otlp-receiver-service.yaml diff --git a/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml new file mode 100644 index 0000000..7f47f4c --- /dev/null +++ b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.sources.otlpReceiver.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-otlp-receiver-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "glassflow-etl.labels" . | nindent 4 }} + app.kubernetes.io/component: otlp-receiver +data: + GLASSFLOW_ROLE: "otlp-receiver" + GLASSFLOW_NATS_SERVER: "{{ .Values.global.nats.address | default (printf "nats://%s-nats.%s.svc.cluster.local:4222" .Release.Name .Release.Namespace) }}" + GLASSFLOW_OTLP_CONFIG_FETCHER_BASE_URL: "http://{{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.api.service.port }}" + GLASSFLOW_LOG_LEVEL: "info" + GLASSFLOW_RUN_LOCAL: "false" +{{- end }} diff --git a/charts/glassflow-etl/templates/otlp-receiver-deployment.yaml b/charts/glassflow-etl/templates/otlp-receiver-deployment.yaml new file mode 100644 index 0000000..e2f37e2 --- /dev/null +++ b/charts/glassflow-etl/templates/otlp-receiver-deployment.yaml @@ -0,0 +1,159 @@ +{{- if .Values.sources.otlpReceiver.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-otlp-receiver + namespace: {{ .Release.Namespace }} + labels: + {{- include "glassflow-etl.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.sources.otlpReceiver.replicas }} + selector: + matchLabels: + {{- include "glassflow-etl.selectorLabels" . | nindent 6 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + app.kubernetes.io/component: otlp-receiver + template: + metadata: + labels: + {{- include "glassflow-etl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: otlp-receiver + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/otlp-receiver-config: {{ include (print $.Template.BasePath "/otlp-receiver-configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + initContainers: + {{- $natsAddress := .Values.global.nats.address | default (printf "nats://%s-nats.%s.svc.cluster.local:4222" .Release.Name .Release.Namespace) }} + - name: wait-for-nats + image: "busybox:1.35" + command: + - sh + - -c + - | + echo "Waiting for NATS to be ready..." + NATS_URL="{{ $natsAddress }}" + # Remove protocol prefix if present + NATS_HOST_PORT="${NATS_URL#nats://}" + NATS_HOST_PORT="${NATS_HOST_PORT#nats-secure://}" + # Extract host and port + if echo "$NATS_HOST_PORT" | grep -q ":"; then + NATS_HOST=$(echo "$NATS_HOST_PORT" | cut -d: -f1) + NATS_PORT=$(echo "$NATS_HOST_PORT" | cut -d: -f2) + else + NATS_HOST="$NATS_HOST_PORT" + NATS_PORT="4222" + fi + echo "Checking NATS at $NATS_HOST:$NATS_PORT..." + until nc -z "$NATS_HOST" "$NATS_PORT"; do + echo "NATS not ready, waiting..." + sleep 2 + done + echo "NATS is ready!" + - name: wait-for-api + image: "busybox:1.35" + command: + - sh + - -c + - | + echo "Waiting for API to be ready..." + until nc -z {{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local {{ .Values.api.service.port }}; do + echo "API not ready, waiting..." + sleep 2 + done + echo "API is ready!" + containers: + - name: otlp-receiver + image: "{{ .Values.global.imageRegistry | default "" }}{{ .Values.sources.otlpReceiver.image.repository }}:{{ .Values.sources.otlpReceiver.image.tag }}" + imagePullPolicy: {{ .Values.sources.otlpReceiver.image.pullPolicy }} + ports: + - name: grpc + containerPort: 4317 + protocol: TCP + - name: http + containerPort: 4318 + protocol: TCP + {{- with .Values.sources.otlpReceiver.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-otlp-receiver-config + env: + {{- if .Values.postgresql.enabled }} + - name: GLASSFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: glassflow-postgresql + key: connection-url + {{- else if .Values.global.postgres.connection_url }} + - name: GLASSFLOW_DATABASE_URL + value: {{ .Values.global.postgres.connection_url | quote }} + {{- else if .Values.global.postgres.secret.name }} + - name: GLASSFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.global.postgres.secret.name }} + key: {{ .Values.global.postgres.secret.key }} + {{- end }} + {{- if .Values.sources.otlpReceiver.env }} + {{- range .Values.sources.otlpReceiver.env }} + - name: {{ .name | quote }} + value: {{ .value | quote }} + {{- end }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 2 + {{- if and .Values.global.encryption.enabled .Values.global.encryption.existingSecret.name }} + volumeMounts: + - name: encryption-key + mountPath: /etc/glassflow/secrets + readOnly: true + {{- end }} + securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + serviceAccountName: {{ include "glassflow-etl.serviceAccountName" . }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if and .Values.global.encryption.enabled .Values.global.encryption.existingSecret.name }} + volumes: + - name: encryption-key + secret: + secretName: {{ .Values.global.encryption.existingSecret.name }} + defaultMode: 0444 + items: + - key: {{ .Values.global.encryption.existingSecret.key | default "encryption-key" }} + path: encryption-key + {{- end }} +{{- end }} diff --git a/charts/glassflow-etl/templates/otlp-receiver-service.yaml b/charts/glassflow-etl/templates/otlp-receiver-service.yaml new file mode 100644 index 0000000..77222b0 --- /dev/null +++ b/charts/glassflow-etl/templates/otlp-receiver-service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.sources.otlpReceiver.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-otlp-receiver + namespace: {{ .Release.Namespace }} + labels: + {{- include "glassflow-etl.labels" . | nindent 4 }} + app.kubernetes.io/component: otlp-receiver +spec: + type: {{ .Values.sources.otlpReceiver.service.type | default "ClusterIP" }} + ports: + - port: {{ .Values.sources.otlpReceiver.service.grpcPort | default 4317 }} + targetPort: grpc + protocol: TCP + name: grpc + - port: {{ .Values.sources.otlpReceiver.service.httpPort | default 4318 }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "glassflow-etl.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: otlp-receiver +{{- end }} From 050c471f8474329fbd09f78bdc9d1e3a6f0c5f58 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Thu, 2 Apr 2026 10:35:23 +0200 Subject: [PATCH 02/13] add missing chart values.yaml --- charts/glassflow-etl/values.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index 116a3fe..1692b10 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -280,6 +280,38 @@ glassflow-operator: className: "" affinity: {} # ============================================================================= +# SOURCES CONFIGURATION +# ============================================================================= +# Data source services that ingest data into GlassFlow pipelines +sources: + # OTLP Receiver - ingests OpenTelemetry data (logs, traces, metrics) via gRPC and HTTP + # This is a shared service (one per installation) that routes data to pipelines via NATS + otlpReceiver: + enabled: false + replicas: 1 + image: + repository: glassflow-etl-otlp-receiver + tag: main + pullPolicy: Always + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + service: + type: ClusterIP + grpcPort: 4317 + httpPort: 4318 + # Environment variables for the OTLP receiver + # Example: + # env: + # - name: GLASSFLOW_LOG_LEVEL + # value: "DEBUG" + env: [] + +# ============================================================================= # NATS OPERATOR CONFIGURATION # ============================================================================= # NATS prometheus exporter for getting metrics from NATS From 5266c4920bdafc515f61f74f75690cafc996c017 Mon Sep 17 00:00:00 2001 From: Kiran Date: Tue, 7 Apr 2026 10:08:36 +0200 Subject: [PATCH 03/13] fix: preserve installation ID on helm upgrade The pre-install/pre-upgrade hook was unconditionally generating a new UUID on every helm upgrade, overwriting the existing installation ID in the secret and sending a spurious helm_install event. The fix reads the existing secret upfront so the ID is reused on upgrades and only generated fresh on first install. --- .../hooks/usage-stats/pre-install.yaml | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml b/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml index a3cfbe7..71eff37 100644 --- a/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml +++ b/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml @@ -35,21 +35,31 @@ spec: set +e # Don't exit on errors - make failures non-blocking apk add --no-cache jq uuidgen >/dev/null 2>&1 - - {{- if .Values.global.usageStats.installationId }} - INSTALLATION_ID="{{ .Values.global.usageStats.installationId }}" - {{- else }} - INSTALLATION_ID=$(uuidgen | tr '[:upper:]' '[:lower:]') - {{- end }} + USAGE_STATS_ENDPOINT="https://tracking.glassflow.dev/api/v1" USAGE_STATS_USERNAME="5PR3vzYGGlYttkBXRTYuzw==" USAGE_STATS_PASSWORD="5tJcYQNR85KKyOwpeIFVdGqGIOgFwxmWEHu0t7WxBbo=" - + KUBE_API="https://kubernetes.default.svc" TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" NAMESPACE="{{ .Release.Namespace }}" SECRET_NAME="{{ .Release.Name }}-usage-stats-config" + + # Read existing secret first — needed both for ID preservation and POST vs PATCH decision + EXISTING_SECRET=$(curl -sS --max-time 5 --cacert "$CA_CERT" \ + -H "Authorization: Bearer $TOKEN" \ + "$KUBE_API/api/v1/namespaces/$NAMESPACE/secrets/$SECRET_NAME" 2>/dev/null || echo "{}") + + {{- if .Values.global.usageStats.installationId }} + INSTALLATION_ID="{{ .Values.global.usageStats.installationId }}" + {{- else }} + # Preserve existing installation ID on upgrades — only generate a new one on first install + INSTALLATION_ID=$(echo "$EXISTING_SECRET" | jq -r '.data.installation_id // empty' | base64 -d 2>/dev/null) + if [ -z "$INSTALLATION_ID" ]; then + INSTALLATION_ID=$(uuidgen | tr '[:upper:]' '[:lower:]') + fi + {{- end }} echo "Usage stats: Getting auth token..." AUTH_RESP=$(curl -sS --max-time 10 -X POST \ @@ -197,11 +207,6 @@ spec: exit 0 fi - # Try to get existing secret first to preserve metadata - EXISTING_SECRET=$(curl -sS --max-time 5 --cacert "$CA_CERT" \ - -H "Authorization: Bearer $TOKEN" \ - "$KUBE_API/api/v1/namespaces/$NAMESPACE/secrets/$SECRET_NAME" 2>/dev/null || echo "{}") - if echo "$EXISTING_SECRET" | jq -e '.kind == "Secret"' >/dev/null 2>&1; then # Secret exists, use PATCH to update it echo "Usage stats: Updating existing secret..." From 9f6b3e6aa222670fcaaa0992b80af2eb39dd22f0 Mon Sep 17 00:00:00 2001 From: Kiran Date: Tue, 14 Apr 2026 17:24:59 +0200 Subject: [PATCH 04/13] feat: integrate pgbouncer subchart into glassflow-etl --- charts/glassflow-etl/Chart.yaml | 7 +- .../glassflow-etl/templates/deployment.yaml | 14 ++- charts/glassflow-etl/values.yaml | 98 +++++++++++++++++++ charts/postgresql/Chart.yaml | 2 +- charts/postgresql/templates/secret.yaml | 13 ++- charts/postgresql/values.yaml | 5 + 6 files changed, 134 insertions(+), 5 deletions(-) diff --git a/charts/glassflow-etl/Chart.yaml b/charts/glassflow-etl/Chart.yaml index e65d033..c16d8a2 100644 --- a/charts/glassflow-etl/Chart.yaml +++ b/charts/glassflow-etl/Chart.yaml @@ -33,10 +33,15 @@ dependencies: condition: nats.enabled alias: nats - name: postgresql - version: "0.1.8" + version: "0.1.9" repository: https://glassflow.github.io/charts condition: postgresql.enabled alias: postgresql + - name: pgbouncer + version: "0.1.0" + repository: https://glassflow.github.io/charts + condition: global.pgbouncer.enabled + alias: pgbouncer maintainers: - name: Glassflow diff --git a/charts/glassflow-etl/templates/deployment.yaml b/charts/glassflow-etl/templates/deployment.yaml index aaf2dae..aeb2617 100644 --- a/charts/glassflow-etl/templates/deployment.yaml +++ b/charts/glassflow-etl/templates/deployment.yaml @@ -217,7 +217,7 @@ spec: sleep 2 done echo "NATS is ready!" - {{- if or .Values.postgresql.enabled .Values.global.postgres.connection_url .Values.global.postgres.secret.name }} + {{- if or .Values.postgresql.enabled .Values.global.postgres.connection_url .Values.global.postgres.direct_connection_url .Values.global.postgres.secret.name }} - name: run-migration image: "{{ .Values.global.imageRegistry | default "" }}glassflow-etl-migration:{{ .Values.api.image.tag }}" imagePullPolicy: {{ .Values.api.image.pullPolicy }} @@ -227,7 +227,15 @@ spec: valueFrom: secretKeyRef: name: glassflow-postgresql + {{- if .Values.global.pgbouncer.enabled }} + # When PGBouncer is enabled, migrations must connect directly to PostgreSQL + key: direct-connection-url + {{- else }} key: connection-url + {{- end }} + {{- else if .Values.global.postgres.direct_connection_url }} + - name: POSTGRES_CONNECTION_URL + value: {{ .Values.global.postgres.direct_connection_url | quote }} {{- else if .Values.global.postgres.connection_url }} - name: POSTGRES_CONNECTION_URL value: {{ .Values.global.postgres.connection_url | quote }} @@ -236,7 +244,11 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.postgres.secret.name }} + {{- if .Values.global.postgres.secret.direct_key }} + key: {{ .Values.global.postgres.secret.direct_key }} + {{- else }} key: {{ .Values.global.postgres.secret.key }} + {{- end }} {{- end }} resources: requests: diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index 742f462..795860a 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -106,11 +106,33 @@ global: postgres: # PostgreSQL connection URL # Format: postgresql://username:password@host:port/database + # Note: when global.pgbouncer.enabled=true, this value is ignored. PGBouncer is only + # supported with the internal PostgreSQL deployment (postgresql.enabled=true). + # For external PostgreSQL, manage your own connection pooler and point + # connection_url to it directly. connection_url: "" + # Optional direct PostgreSQL URL for migrations (DDL). + # Recommended when connection_url points to a pooler such as PgBouncer. + # If empty, migration falls back to connection_url. + direct_connection_url: "" # Alternatively, source the connection URL from a Kubernetes secret secret: name: "" + # App/runtime connection URL key key: "connection_url" + # Optional direct migration URL key + # If empty, migration falls back to "key". + direct_key: "" + + # PGBouncer toggle — this is the single flag to enable/disable PGBouncer. + # Setting this to true: + # - Deploys the PGBouncer subchart + # - Configures the glassflow-postgresql secret so that connection-url points to + # PGBouncer instead of PostgreSQL directly (transparent to all components) + # - Routes DB migrations to PostgreSQL directly via direct-connection-url + # Only supported with postgresql.enabled=true (internal PostgreSQL). + pgbouncer: + enabled: false # Encryption configuration for connection credentials encryption: @@ -400,6 +422,82 @@ postgresql: memory: "2Gi" cpu: "1000m" +# ============================================================================= +# PGBOUNCER CONFIGURATION +# ============================================================================= +# PGBouncer connection pooler configuration. +# +# Enable via: --set global.pgbouncer.enabled=true +# +# When enabled, PGBouncer is deployed and all pipeline components, the API, +# and the operator connect to PostgreSQL through it. This prevents connection +# exhaustion when many pipeline replicas are running (e.g. schema evolution with +# many ingestor/sink replicas each holding their own connection pool). +# +# Requirements: +# - postgresql.enabled must be true (internal PostgreSQL). +# For external PostgreSQL, manage your own connection pooler and point +# global.postgres.connection_url to it directly. +# +# Existing users are not affected: defaults to disabled (global.pgbouncer.enabled=false). +pgbouncer: + # Database alias exposed by pgbouncer to clients. + # Must match the database name used in glassflow-postgresql connection-url. + databaseAlias: "glassflow" + + image: + repository: bitnamilegacy/pgbouncer + tag: "1.23.1" + pullPolicy: IfNotPresent + + # Pool mode: transaction (recommended) or session. + # transaction - server connection is returned to the pool after each transaction. + # Most efficient. Requires QueryExecModeSimpleProtocol in the glassflow-api + # postgres driver, which is set automatically when pgbouncer is enabled. + # session - server connection held for the entire client session. + # Compatible with prepared statements but provides less pooling benefit. + poolMode: "transaction" + + # Maximum number of client connections PGBouncer will accept + maxClientConn: 1000 + + # Number of server connections per user/database pair + defaultPoolSize: 20 + + # Minimum number of server connections kept open (0 = none) + minPoolSize: 0 + + # Extra server connections to use when pool is exhausted + reservePoolSize: 5 + + # Seconds a server connection can be idle before being closed (0 = disable) + serverIdleTimeout: 600 + + # Seconds a client connection can be idle before being closed (0 = disable) + clientIdleTimeout: 0 + + # Startup parameters to silently ignore (pgx sends extra_float_digits by default) + ignoreStartupParameters: "extra_float_digits" + + # Server reset query run when a connection is returned to the pool. + # Leave empty for transaction mode. Set to "DISCARD ALL" for session mode. + serverResetQuery: "" + + replicaCount: 1 + + service: + type: ClusterIP + port: 5432 + targetPort: 5432 + + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "200m" + # ============================================================================= # NATS CONFIGURATION # ============================================================================= diff --git a/charts/postgresql/Chart.yaml b/charts/postgresql/Chart.yaml index 2a134e1..6955469 100644 --- a/charts/postgresql/Chart.yaml +++ b/charts/postgresql/Chart.yaml @@ -6,7 +6,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.1.8 +version: 0.1.9 # This is the version number of the application being deployed. appVersion: "17" diff --git a/charts/postgresql/templates/secret.yaml b/charts/postgresql/templates/secret.yaml index 786d06c..44c18c5 100644 --- a/charts/postgresql/templates/secret.yaml +++ b/charts/postgresql/templates/secret.yaml @@ -28,10 +28,19 @@ data: {{- $username = printf "pguser_%s" (randAlphaNum 8) }} {{- end }} {{- $sslmode := .Values.auth.sslmode | default "disable" }} -{{- $connectionUrl := printf "postgresql://%s:%s@%s:%d/%s?sslmode=%s" $username $userPassword $host $port $database $sslmode }} +{{- $directConnectionUrl := printf "postgresql://%s:%s@%s:%d/%s?sslmode=%s" $username $userPassword $host $port $database $sslmode }} # Credentials (only created when not using existing secret) username: {{ $username | b64enc | quote }} password: {{ $userPassword | b64enc | quote }} - connection-url: {{ $connectionUrl | b64enc | quote }} + {{- if .Values.global.pgbouncer.enabled }} + {{- $pgbouncerHost := printf "%s-pgbouncer.%s.svc.cluster.local" .Release.Name .Release.Namespace }} + {{- $pgbouncerUrl := printf "postgresql://%s:%s@%s:5432/%s?sslmode=disable" $username $userPassword $pgbouncerHost $database }} + # When pgbouncer is enabled, connection-url points to PGBouncer (used by API, operator, and pipeline components). + # direct-connection-url points to PostgreSQL directly (used by DB migrations). + connection-url: {{ $pgbouncerUrl | b64enc | quote }} + direct-connection-url: {{ $directConnectionUrl | b64enc | quote }} + {{- else }} + connection-url: {{ $directConnectionUrl | b64enc | quote }} + {{- end }} {{- end }} diff --git a/charts/postgresql/values.yaml b/charts/postgresql/values.yaml index 53339f9..816525c 100644 --- a/charts/postgresql/values.yaml +++ b/charts/postgresql/values.yaml @@ -9,6 +9,11 @@ global: # Global storage class - if set, will be used for persistent volumes storageClass: "" + pgbouncer: + # When true, connection-url in the glassflow-postgresql secret points to PGBouncer + # instead of PostgreSQL directly. This is set automatically by the glassflow-etl chart + # when global.pgbouncer.enabled=true and should not be set manually. + enabled: false # ============================================================================= # IMAGE CONFIGURATION From 1114977a585d57410e55b0576c65c97e24a65f75 Mon Sep 17 00:00:00 2001 From: Kiran Date: Tue, 14 Apr 2026 17:33:07 +0200 Subject: [PATCH 05/13] fix: add helm repo add steps to lint workflow --- .github/workflows/lint.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d72da38..13e24a8 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -27,6 +27,13 @@ jobs: - name: Set up chart-testing uses: helm/chart-testing-action@v2.6.1 + - name: Add Helm repositories + run: | + helm repo add nats https://nats-io.github.io/k8s/helm/charts/ + helm repo add glassflow-operator https://glassflow.github.io/glassflow-etl-k8s-operator + helm repo add glassflow-charts https://glassflow.github.io/charts + helm repo update + - name: Update Helm dependencies run: helm dependency update charts/glassflow-etl From 0d8a5f8f6f3840219716b86a28bd834e828463f1 Mon Sep 17 00:00:00 2001 From: Kiran Date: Tue, 14 Apr 2026 17:38:20 +0200 Subject: [PATCH 06/13] revert: postgres chart version change --- charts/glassflow-etl/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/glassflow-etl/Chart.yaml b/charts/glassflow-etl/Chart.yaml index c16d8a2..4b8bcc3 100644 --- a/charts/glassflow-etl/Chart.yaml +++ b/charts/glassflow-etl/Chart.yaml @@ -33,7 +33,7 @@ dependencies: condition: nats.enabled alias: nats - name: postgresql - version: "0.1.9" + version: "0.1.8" repository: https://glassflow.github.io/charts condition: postgresql.enabled alias: postgresql From 0fb416b23abb97e570afbc7f0caa0a2c9ffdf37b Mon Sep 17 00:00:00 2001 From: Kiran Date: Tue, 14 Apr 2026 17:53:58 +0200 Subject: [PATCH 07/13] feat: update operator chart --- charts/glassflow-etl/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/glassflow-etl/Chart.yaml b/charts/glassflow-etl/Chart.yaml index 4b8bcc3..2230791 100644 --- a/charts/glassflow-etl/Chart.yaml +++ b/charts/glassflow-etl/Chart.yaml @@ -25,7 +25,7 @@ appVersion: "2.11.2" dependencies: - name: glassflow-operator - version: "0.7.8" + version: "0.8.0" repository: https://glassflow.github.io/glassflow-etl-k8s-operator - name: nats version: "2.12.3" From ffd8951a6cfdce821756be48c152b92cbdd1f188 Mon Sep 17 00:00:00 2001 From: Kiran Date: Tue, 14 Apr 2026 17:55:25 +0200 Subject: [PATCH 08/13] revert: linter job changes --- .github/workflows/lint.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 13e24a8..d72da38 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -27,13 +27,6 @@ jobs: - name: Set up chart-testing uses: helm/chart-testing-action@v2.6.1 - - name: Add Helm repositories - run: | - helm repo add nats https://nats-io.github.io/k8s/helm/charts/ - helm repo add glassflow-operator https://glassflow.github.io/glassflow-etl-k8s-operator - helm repo add glassflow-charts https://glassflow.github.io/charts - helm repo update - - name: Update Helm dependencies run: helm dependency update charts/glassflow-etl From 20680e616cf4fef99611187bbda7e28434923ed7 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 17 Apr 2026 15:32:04 +0900 Subject: [PATCH 09/13] add OTEL_EXPORTER_OTLP_ENDPOINT to otlp receiver --- charts/glassflow-etl/templates/otlp-receiver-configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml index 7f47f4c..382e74d 100644 --- a/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml +++ b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml @@ -13,4 +13,5 @@ data: GLASSFLOW_OTLP_CONFIG_FETCHER_BASE_URL: "http://{{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.api.service.port }}" GLASSFLOW_LOG_LEVEL: "info" GLASSFLOW_RUN_LOCAL: "false" + OTEL_EXPORTER_OTLP_ENDPOINT: "http://{{ .Release.Name }}-otel-collector.{{ .Release.Namespace }}.svc.cluster.local:4318" {{- end }} From 96b80d6b6b794f5d14d2c8cb2f58eddf5d3ac621 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Tue, 21 Apr 2026 08:58:00 +0200 Subject: [PATCH 10/13] bump notification service version --- charts/glassflow-etl/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index cb71ed8..97e9ccf 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -705,7 +705,7 @@ notificationService: replicas: 1 image: repository: glassflow-notifier - tag: v1.0.1 + tag: v1.0.2 pullPolicy: IfNotPresent resources: requests: From 7abc368175fda37cd9ba8bb2788473d86b4ca2d4 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Tue, 21 Apr 2026 09:15:35 +0200 Subject: [PATCH 11/13] Set notifications enabled by default --- charts/glassflow-etl/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index 97e9ccf..ef7241a 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -78,7 +78,7 @@ global: cpu: 500m memory: 512Mi notifications: # configure slack / email channel details in notificationService section below - enabled: false + enabled: true nats: # NATS address for the operator to connect to # Defaults to {{ .Release.Name }}-nats.{{ .Release.Namespace }}.svc.cluster.local:4222 if not specified From b5e13b580cb7040aff43805f5a39e0be6e689778 Mon Sep 17 00:00:00 2001 From: Kiran Date: Thu, 23 Apr 2026 15:40:04 +0200 Subject: [PATCH 12/13] fix: linter error --- charts/postgresql/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/postgresql/values.yaml b/charts/postgresql/values.yaml index 3401071..816525c 100644 --- a/charts/postgresql/values.yaml +++ b/charts/postgresql/values.yaml @@ -213,4 +213,4 @@ initContainers: [] additionalVolumeMounts: [] # Additional volumes -additionalVolumes: [] \ No newline at end of file +additionalVolumes: [] From 5ab38604059d23241c3b20e110bb3d649aa3cfa3 Mon Sep 17 00:00:00 2001 From: Kiran Date: Thu, 23 Apr 2026 15:44:56 +0200 Subject: [PATCH 13/13] feat: update app and chart version --- charts/glassflow-etl/Chart.yaml | 6 +++--- charts/glassflow-etl/values.yaml | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/charts/glassflow-etl/Chart.yaml b/charts/glassflow-etl/Chart.yaml index 2230791..6ae1683 100644 --- a/charts/glassflow-etl/Chart.yaml +++ b/charts/glassflow-etl/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.15 +version: 0.5.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.11.2" +appVersion: "3.0.0" dependencies: - name: glassflow-operator @@ -33,7 +33,7 @@ dependencies: condition: nats.enabled alias: nats - name: postgresql - version: "0.1.8" + version: "0.1.9" repository: https://glassflow.github.io/charts condition: postgresql.enabled alias: postgresql diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index ef7241a..2f43906 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -152,7 +152,7 @@ api: logLevel: "info" image: repository: glassflow-etl-be - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent resources: requests: @@ -187,7 +187,7 @@ ui: replicas: 1 image: repository: glassflow-etl-fe - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "info" resources: @@ -266,7 +266,7 @@ glassflow-operator: reconcileTimeout: 15m image: repository: glassflow-etl-k8s-operator - tag: v2.1.0 + tag: v3.0.0 pullPolicy: IfNotPresent resources: limits: @@ -282,7 +282,7 @@ glassflow-operator: ingestor: image: repository: glassflow-etl-ingestor - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -297,7 +297,7 @@ glassflow-operator: join: image: repository: glassflow-etl-join - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -312,7 +312,7 @@ glassflow-operator: sink: image: repository: glassflow-etl-sink - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -327,7 +327,7 @@ glassflow-operator: dedup: image: repository: glassflow-etl-dedup - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -353,7 +353,7 @@ sources: replicas: 1 image: repository: glassflow-etl-otlp-receiver - tag: main + tag: v3.0.0 pullPolicy: Always resources: requests: