diff --git a/charts/glassflow-etl/Chart.yaml b/charts/glassflow-etl/Chart.yaml index e65d033..6ae1683 100644 --- a/charts/glassflow-etl/Chart.yaml +++ b/charts/glassflow-etl/Chart.yaml @@ -15,17 +15,17 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.15 +version: 0.5.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.11.2" +appVersion: "3.0.0" dependencies: - name: glassflow-operator - version: "0.7.8" + version: "0.8.0" repository: https://glassflow.github.io/glassflow-etl-k8s-operator - name: nats version: "2.12.3" @@ -33,10 +33,15 @@ dependencies: condition: nats.enabled alias: nats - name: postgresql - version: "0.1.8" + version: "0.1.9" repository: https://glassflow.github.io/charts condition: postgresql.enabled alias: postgresql + - name: pgbouncer + version: "0.1.0" + repository: https://glassflow.github.io/charts + condition: global.pgbouncer.enabled + alias: pgbouncer maintainers: - name: Glassflow diff --git a/charts/glassflow-etl/templates/deployment.yaml b/charts/glassflow-etl/templates/deployment.yaml index aaf2dae..aeb2617 100644 --- a/charts/glassflow-etl/templates/deployment.yaml +++ b/charts/glassflow-etl/templates/deployment.yaml @@ -217,7 +217,7 @@ spec: sleep 2 done echo "NATS is ready!" - {{- if or .Values.postgresql.enabled .Values.global.postgres.connection_url .Values.global.postgres.secret.name }} + {{- if or .Values.postgresql.enabled .Values.global.postgres.connection_url .Values.global.postgres.direct_connection_url .Values.global.postgres.secret.name }} - name: run-migration image: "{{ .Values.global.imageRegistry | default "" }}glassflow-etl-migration:{{ .Values.api.image.tag }}" imagePullPolicy: {{ .Values.api.image.pullPolicy }} @@ -227,7 +227,15 @@ spec: valueFrom: secretKeyRef: name: glassflow-postgresql + {{- if .Values.global.pgbouncer.enabled }} + # When PGBouncer is enabled, migrations must connect directly to PostgreSQL + key: direct-connection-url + {{- else }} key: connection-url + {{- end }} + {{- else if .Values.global.postgres.direct_connection_url }} + - name: POSTGRES_CONNECTION_URL + value: {{ .Values.global.postgres.direct_connection_url | quote }} {{- else if .Values.global.postgres.connection_url }} - name: POSTGRES_CONNECTION_URL value: {{ .Values.global.postgres.connection_url | quote }} @@ -236,7 +244,11 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.postgres.secret.name }} + {{- if .Values.global.postgres.secret.direct_key }} + key: {{ .Values.global.postgres.secret.direct_key }} + {{- else }} key: {{ .Values.global.postgres.secret.key }} + {{- end }} {{- end }} resources: requests: diff --git a/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml b/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml index a3cfbe7..71eff37 100644 --- a/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml +++ b/charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml @@ -35,21 +35,31 @@ spec: set +e # Don't exit on errors - make failures non-blocking apk add --no-cache jq uuidgen >/dev/null 2>&1 - - {{- if .Values.global.usageStats.installationId }} - INSTALLATION_ID="{{ .Values.global.usageStats.installationId }}" - {{- else }} - INSTALLATION_ID=$(uuidgen | tr '[:upper:]' '[:lower:]') - {{- end }} + USAGE_STATS_ENDPOINT="https://tracking.glassflow.dev/api/v1" USAGE_STATS_USERNAME="5PR3vzYGGlYttkBXRTYuzw==" USAGE_STATS_PASSWORD="5tJcYQNR85KKyOwpeIFVdGqGIOgFwxmWEHu0t7WxBbo=" - + KUBE_API="https://kubernetes.default.svc" TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" NAMESPACE="{{ .Release.Namespace }}" SECRET_NAME="{{ .Release.Name }}-usage-stats-config" + + # Read existing secret first — needed both for ID preservation and POST vs PATCH decision + EXISTING_SECRET=$(curl -sS --max-time 5 --cacert "$CA_CERT" \ + -H "Authorization: Bearer $TOKEN" \ + "$KUBE_API/api/v1/namespaces/$NAMESPACE/secrets/$SECRET_NAME" 2>/dev/null || echo "{}") + + {{- if .Values.global.usageStats.installationId }} + INSTALLATION_ID="{{ .Values.global.usageStats.installationId }}" + {{- else }} + # Preserve existing installation ID on upgrades — only generate a new one on first install + INSTALLATION_ID=$(echo "$EXISTING_SECRET" | jq -r '.data.installation_id // empty' | base64 -d 2>/dev/null) + if [ -z "$INSTALLATION_ID" ]; then + INSTALLATION_ID=$(uuidgen | tr '[:upper:]' '[:lower:]') + fi + {{- end }} echo "Usage stats: Getting auth token..." AUTH_RESP=$(curl -sS --max-time 10 -X POST \ @@ -197,11 +207,6 @@ spec: exit 0 fi - # Try to get existing secret first to preserve metadata - EXISTING_SECRET=$(curl -sS --max-time 5 --cacert "$CA_CERT" \ - -H "Authorization: Bearer $TOKEN" \ - "$KUBE_API/api/v1/namespaces/$NAMESPACE/secrets/$SECRET_NAME" 2>/dev/null || echo "{}") - if echo "$EXISTING_SECRET" | jq -e '.kind == "Secret"' >/dev/null 2>&1; then # Secret exists, use PATCH to update it echo "Usage stats: Updating existing secret..." diff --git a/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml new file mode 100644 index 0000000..382e74d --- /dev/null +++ b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.sources.otlpReceiver.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-otlp-receiver-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "glassflow-etl.labels" . | nindent 4 }} + app.kubernetes.io/component: otlp-receiver +data: + GLASSFLOW_ROLE: "otlp-receiver" + GLASSFLOW_NATS_SERVER: "{{ .Values.global.nats.address | default (printf "nats://%s-nats.%s.svc.cluster.local:4222" .Release.Name .Release.Namespace) }}" + GLASSFLOW_OTLP_CONFIG_FETCHER_BASE_URL: "http://{{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.api.service.port }}" + GLASSFLOW_LOG_LEVEL: "info" + GLASSFLOW_RUN_LOCAL: "false" + OTEL_EXPORTER_OTLP_ENDPOINT: "http://{{ .Release.Name }}-otel-collector.{{ .Release.Namespace }}.svc.cluster.local:4318" +{{- end }} diff --git a/charts/glassflow-etl/templates/otlp-receiver-deployment.yaml b/charts/glassflow-etl/templates/otlp-receiver-deployment.yaml new file mode 100644 index 0000000..e2f37e2 --- /dev/null +++ b/charts/glassflow-etl/templates/otlp-receiver-deployment.yaml @@ -0,0 +1,159 @@ +{{- if .Values.sources.otlpReceiver.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-otlp-receiver + namespace: {{ .Release.Namespace }} + labels: + {{- include "glassflow-etl.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.sources.otlpReceiver.replicas }} + selector: + matchLabels: + {{- include "glassflow-etl.selectorLabels" . | nindent 6 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + app.kubernetes.io/component: otlp-receiver + template: + metadata: + labels: + {{- include "glassflow-etl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: otlp-receiver + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/otlp-receiver-config: {{ include (print $.Template.BasePath "/otlp-receiver-configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + initContainers: + {{- $natsAddress := .Values.global.nats.address | default (printf "nats://%s-nats.%s.svc.cluster.local:4222" .Release.Name .Release.Namespace) }} + - name: wait-for-nats + image: "busybox:1.35" + command: + - sh + - -c + - | + echo "Waiting for NATS to be ready..." + NATS_URL="{{ $natsAddress }}" + # Remove protocol prefix if present + NATS_HOST_PORT="${NATS_URL#nats://}" + NATS_HOST_PORT="${NATS_HOST_PORT#nats-secure://}" + # Extract host and port + if echo "$NATS_HOST_PORT" | grep -q ":"; then + NATS_HOST=$(echo "$NATS_HOST_PORT" | cut -d: -f1) + NATS_PORT=$(echo "$NATS_HOST_PORT" | cut -d: -f2) + else + NATS_HOST="$NATS_HOST_PORT" + NATS_PORT="4222" + fi + echo "Checking NATS at $NATS_HOST:$NATS_PORT..." + until nc -z "$NATS_HOST" "$NATS_PORT"; do + echo "NATS not ready, waiting..." + sleep 2 + done + echo "NATS is ready!" + - name: wait-for-api + image: "busybox:1.35" + command: + - sh + - -c + - | + echo "Waiting for API to be ready..." + until nc -z {{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local {{ .Values.api.service.port }}; do + echo "API not ready, waiting..." + sleep 2 + done + echo "API is ready!" + containers: + - name: otlp-receiver + image: "{{ .Values.global.imageRegistry | default "" }}{{ .Values.sources.otlpReceiver.image.repository }}:{{ .Values.sources.otlpReceiver.image.tag }}" + imagePullPolicy: {{ .Values.sources.otlpReceiver.image.pullPolicy }} + ports: + - name: grpc + containerPort: 4317 + protocol: TCP + - name: http + containerPort: 4318 + protocol: TCP + {{- with .Values.sources.otlpReceiver.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-otlp-receiver-config + env: + {{- if .Values.postgresql.enabled }} + - name: GLASSFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: glassflow-postgresql + key: connection-url + {{- else if .Values.global.postgres.connection_url }} + - name: GLASSFLOW_DATABASE_URL + value: {{ .Values.global.postgres.connection_url | quote }} + {{- else if .Values.global.postgres.secret.name }} + - name: GLASSFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.global.postgres.secret.name }} + key: {{ .Values.global.postgres.secret.key }} + {{- end }} + {{- if .Values.sources.otlpReceiver.env }} + {{- range .Values.sources.otlpReceiver.env }} + - name: {{ .name | quote }} + value: {{ .value | quote }} + {{- end }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 2 + {{- if and .Values.global.encryption.enabled .Values.global.encryption.existingSecret.name }} + volumeMounts: + - name: encryption-key + mountPath: /etc/glassflow/secrets + readOnly: true + {{- end }} + securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + serviceAccountName: {{ include "glassflow-etl.serviceAccountName" . }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if and .Values.global.encryption.enabled .Values.global.encryption.existingSecret.name }} + volumes: + - name: encryption-key + secret: + secretName: {{ .Values.global.encryption.existingSecret.name }} + defaultMode: 0444 + items: + - key: {{ .Values.global.encryption.existingSecret.key | default "encryption-key" }} + path: encryption-key + {{- end }} +{{- end }} diff --git a/charts/glassflow-etl/templates/otlp-receiver-service.yaml b/charts/glassflow-etl/templates/otlp-receiver-service.yaml new file mode 100644 index 0000000..77222b0 --- /dev/null +++ b/charts/glassflow-etl/templates/otlp-receiver-service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.sources.otlpReceiver.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-otlp-receiver + namespace: {{ .Release.Namespace }} + labels: + {{- include "glassflow-etl.labels" . | nindent 4 }} + app.kubernetes.io/component: otlp-receiver +spec: + type: {{ .Values.sources.otlpReceiver.service.type | default "ClusterIP" }} + ports: + - port: {{ .Values.sources.otlpReceiver.service.grpcPort | default 4317 }} + targetPort: grpc + protocol: TCP + name: grpc + - port: {{ .Values.sources.otlpReceiver.service.httpPort | default 4318 }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "glassflow-etl.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: otlp-receiver +{{- end }} diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index 742f462..2f43906 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -78,7 +78,7 @@ global: cpu: 500m memory: 512Mi notifications: # configure slack / email channel details in notificationService section below - enabled: false + enabled: true nats: # NATS address for the operator to connect to # Defaults to {{ .Release.Name }}-nats.{{ .Release.Namespace }}.svc.cluster.local:4222 if not specified @@ -106,11 +106,33 @@ global: postgres: # PostgreSQL connection URL # Format: postgresql://username:password@host:port/database + # Note: when global.pgbouncer.enabled=true, this value is ignored. PGBouncer is only + # supported with the internal PostgreSQL deployment (postgresql.enabled=true). + # For external PostgreSQL, manage your own connection pooler and point + # connection_url to it directly. connection_url: "" + # Optional direct PostgreSQL URL for migrations (DDL). + # Recommended when connection_url points to a pooler such as PgBouncer. + # If empty, migration falls back to connection_url. + direct_connection_url: "" # Alternatively, source the connection URL from a Kubernetes secret secret: name: "" + # App/runtime connection URL key key: "connection_url" + # Optional direct migration URL key + # If empty, migration falls back to "key". + direct_key: "" + + # PGBouncer toggle — this is the single flag to enable/disable PGBouncer. + # Setting this to true: + # - Deploys the PGBouncer subchart + # - Configures the glassflow-postgresql secret so that connection-url points to + # PGBouncer instead of PostgreSQL directly (transparent to all components) + # - Routes DB migrations to PostgreSQL directly via direct-connection-url + # Only supported with postgresql.enabled=true (internal PostgreSQL). + pgbouncer: + enabled: false # Encryption configuration for connection credentials encryption: @@ -130,7 +152,7 @@ api: logLevel: "info" image: repository: glassflow-etl-be - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent resources: requests: @@ -165,7 +187,7 @@ ui: replicas: 1 image: repository: glassflow-etl-fe - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "info" resources: @@ -244,7 +266,7 @@ glassflow-operator: reconcileTimeout: 15m image: repository: glassflow-etl-k8s-operator - tag: v2.1.0 + tag: v3.0.0 pullPolicy: IfNotPresent resources: limits: @@ -260,7 +282,7 @@ glassflow-operator: ingestor: image: repository: glassflow-etl-ingestor - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -275,7 +297,7 @@ glassflow-operator: join: image: repository: glassflow-etl-join - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -290,7 +312,7 @@ glassflow-operator: sink: image: repository: glassflow-etl-sink - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -305,7 +327,7 @@ glassflow-operator: dedup: image: repository: glassflow-etl-dedup - tag: v2.11.2 + tag: v3.0.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -320,6 +342,38 @@ glassflow-operator: className: "" affinity: {} # ============================================================================= +# SOURCES CONFIGURATION +# ============================================================================= +# Data source services that ingest data into GlassFlow pipelines +sources: + # OTLP Receiver - ingests OpenTelemetry data (logs, traces, metrics) via gRPC and HTTP + # This is a shared service (one per installation) that routes data to pipelines via NATS + otlpReceiver: + enabled: false + replicas: 1 + image: + repository: glassflow-etl-otlp-receiver + tag: v3.0.0 + pullPolicy: Always + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + service: + type: ClusterIP + grpcPort: 4317 + httpPort: 4318 + # Environment variables for the OTLP receiver + # Example: + # env: + # - name: GLASSFLOW_LOG_LEVEL + # value: "DEBUG" + env: [] + +# ============================================================================= # NATS OPERATOR CONFIGURATION # ============================================================================= # NATS prometheus exporter for getting metrics from NATS @@ -400,6 +454,82 @@ postgresql: memory: "2Gi" cpu: "1000m" +# ============================================================================= +# PGBOUNCER CONFIGURATION +# ============================================================================= +# PGBouncer connection pooler configuration. +# +# Enable via: --set global.pgbouncer.enabled=true +# +# When enabled, PGBouncer is deployed and all pipeline components, the API, +# and the operator connect to PostgreSQL through it. This prevents connection +# exhaustion when many pipeline replicas are running (e.g. schema evolution with +# many ingestor/sink replicas each holding their own connection pool). +# +# Requirements: +# - postgresql.enabled must be true (internal PostgreSQL). +# For external PostgreSQL, manage your own connection pooler and point +# global.postgres.connection_url to it directly. +# +# Existing users are not affected: defaults to disabled (global.pgbouncer.enabled=false). +pgbouncer: + # Database alias exposed by pgbouncer to clients. + # Must match the database name used in glassflow-postgresql connection-url. + databaseAlias: "glassflow" + + image: + repository: bitnamilegacy/pgbouncer + tag: "1.23.1" + pullPolicy: IfNotPresent + + # Pool mode: transaction (recommended) or session. + # transaction - server connection is returned to the pool after each transaction. + # Most efficient. Requires QueryExecModeSimpleProtocol in the glassflow-api + # postgres driver, which is set automatically when pgbouncer is enabled. + # session - server connection held for the entire client session. + # Compatible with prepared statements but provides less pooling benefit. + poolMode: "transaction" + + # Maximum number of client connections PGBouncer will accept + maxClientConn: 1000 + + # Number of server connections per user/database pair + defaultPoolSize: 20 + + # Minimum number of server connections kept open (0 = none) + minPoolSize: 0 + + # Extra server connections to use when pool is exhausted + reservePoolSize: 5 + + # Seconds a server connection can be idle before being closed (0 = disable) + serverIdleTimeout: 600 + + # Seconds a client connection can be idle before being closed (0 = disable) + clientIdleTimeout: 0 + + # Startup parameters to silently ignore (pgx sends extra_float_digits by default) + ignoreStartupParameters: "extra_float_digits" + + # Server reset query run when a connection is returned to the pool. + # Leave empty for transaction mode. Set to "DISCARD ALL" for session mode. + serverResetQuery: "" + + replicaCount: 1 + + service: + type: ClusterIP + port: 5432 + targetPort: 5432 + + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "200m" + # ============================================================================= # NATS CONFIGURATION # ============================================================================= @@ -575,7 +705,7 @@ notificationService: replicas: 1 image: repository: glassflow-notifier - tag: v1.0.1 + tag: v1.0.2 pullPolicy: IfNotPresent resources: requests: diff --git a/charts/postgresql/values.yaml b/charts/postgresql/values.yaml index 3401071..816525c 100644 --- a/charts/postgresql/values.yaml +++ b/charts/postgresql/values.yaml @@ -213,4 +213,4 @@ initContainers: [] additionalVolumeMounts: [] # Additional volumes -additionalVolumes: [] \ No newline at end of file +additionalVolumes: []