Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions charts/glassflow-etl/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,33 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.5.15
version: 0.5.16

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "2.11.2"
appVersion: "3.0.0"

dependencies:
- name: glassflow-operator
version: "0.7.8"
version: "0.8.0"
repository: https://glassflow.github.io/glassflow-etl-k8s-operator
- name: nats
version: "2.12.3"
repository: https://nats-io.github.io/k8s/helm/charts/
condition: nats.enabled
alias: nats
- name: postgresql
version: "0.1.8"
version: "0.1.9"
repository: https://glassflow.github.io/charts
condition: postgresql.enabled
alias: postgresql
- name: pgbouncer
version: "0.1.0"
repository: https://glassflow.github.io/charts
condition: global.pgbouncer.enabled
alias: pgbouncer

maintainers:
- name: Glassflow
Expand Down
14 changes: 13 additions & 1 deletion charts/glassflow-etl/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ spec:
sleep 2
done
echo "NATS is ready!"
{{- if or .Values.postgresql.enabled .Values.global.postgres.connection_url .Values.global.postgres.secret.name }}
{{- if or .Values.postgresql.enabled .Values.global.postgres.connection_url .Values.global.postgres.direct_connection_url .Values.global.postgres.secret.name }}
- name: run-migration
image: "{{ .Values.global.imageRegistry | default "" }}glassflow-etl-migration:{{ .Values.api.image.tag }}"
imagePullPolicy: {{ .Values.api.image.pullPolicy }}
Expand All @@ -227,7 +227,15 @@ spec:
valueFrom:
secretKeyRef:
name: glassflow-postgresql
{{- if .Values.global.pgbouncer.enabled }}
# When PGBouncer is enabled, migrations must connect directly to PostgreSQL
key: direct-connection-url
{{- else }}
key: connection-url
{{- end }}
{{- else if .Values.global.postgres.direct_connection_url }}
- name: POSTGRES_CONNECTION_URL
value: {{ .Values.global.postgres.direct_connection_url | quote }}
{{- else if .Values.global.postgres.connection_url }}
- name: POSTGRES_CONNECTION_URL
value: {{ .Values.global.postgres.connection_url | quote }}
Expand All @@ -236,7 +244,11 @@ spec:
valueFrom:
secretKeyRef:
name: {{ .Values.global.postgres.secret.name }}
{{- if .Values.global.postgres.secret.direct_key }}
key: {{ .Values.global.postgres.secret.direct_key }}
{{- else }}
key: {{ .Values.global.postgres.secret.key }}
{{- end }}
{{- end }}
resources:
requests:
Expand Down
29 changes: 17 additions & 12 deletions charts/glassflow-etl/templates/hooks/usage-stats/pre-install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,31 @@ spec:
set +e # Don't exit on errors - make failures non-blocking

apk add --no-cache jq uuidgen >/dev/null 2>&1

{{- if .Values.global.usageStats.installationId }}
INSTALLATION_ID="{{ .Values.global.usageStats.installationId }}"
{{- else }}
INSTALLATION_ID=$(uuidgen | tr '[:upper:]' '[:lower:]')
{{- end }}

USAGE_STATS_ENDPOINT="https://tracking.glassflow.dev/api/v1"
USAGE_STATS_USERNAME="5PR3vzYGGlYttkBXRTYuzw=="
USAGE_STATS_PASSWORD="5tJcYQNR85KKyOwpeIFVdGqGIOgFwxmWEHu0t7WxBbo="

KUBE_API="https://kubernetes.default.svc"
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
NAMESPACE="{{ .Release.Namespace }}"
SECRET_NAME="{{ .Release.Name }}-usage-stats-config"

# Read existing secret first — needed both for ID preservation and POST vs PATCH decision
EXISTING_SECRET=$(curl -sS --max-time 5 --cacert "$CA_CERT" \
-H "Authorization: Bearer $TOKEN" \
"$KUBE_API/api/v1/namespaces/$NAMESPACE/secrets/$SECRET_NAME" 2>/dev/null || echo "{}")

{{- if .Values.global.usageStats.installationId }}
INSTALLATION_ID="{{ .Values.global.usageStats.installationId }}"
{{- else }}
# Preserve existing installation ID on upgrades — only generate a new one on first install
INSTALLATION_ID=$(echo "$EXISTING_SECRET" | jq -r '.data.installation_id // empty' | base64 -d 2>/dev/null)
if [ -z "$INSTALLATION_ID" ]; then
INSTALLATION_ID=$(uuidgen | tr '[:upper:]' '[:lower:]')
fi
{{- end }}

echo "Usage stats: Getting auth token..."
AUTH_RESP=$(curl -sS --max-time 10 -X POST \
Expand Down Expand Up @@ -197,11 +207,6 @@ spec:
exit 0
fi

# Try to get existing secret first to preserve metadata
EXISTING_SECRET=$(curl -sS --max-time 5 --cacert "$CA_CERT" \
-H "Authorization: Bearer $TOKEN" \
"$KUBE_API/api/v1/namespaces/$NAMESPACE/secrets/$SECRET_NAME" 2>/dev/null || echo "{}")

if echo "$EXISTING_SECRET" | jq -e '.kind == "Secret"' >/dev/null 2>&1; then
# Secret exists, use PATCH to update it
echo "Usage stats: Updating existing secret..."
Expand Down
17 changes: 17 additions & 0 deletions charts/glassflow-etl/templates/otlp-receiver-configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{{- if .Values.sources.otlpReceiver.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Release.Name }}-otlp-receiver-config
namespace: {{ .Release.Namespace }}
labels:
{{- include "glassflow-etl.labels" . | nindent 4 }}
app.kubernetes.io/component: otlp-receiver
data:
GLASSFLOW_ROLE: "otlp-receiver"
GLASSFLOW_NATS_SERVER: "{{ .Values.global.nats.address | default (printf "nats://%s-nats.%s.svc.cluster.local:4222" .Release.Name .Release.Namespace) }}"
GLASSFLOW_OTLP_CONFIG_FETCHER_BASE_URL: "http://{{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.api.service.port }}"
GLASSFLOW_LOG_LEVEL: "info"
GLASSFLOW_RUN_LOCAL: "false"
OTEL_EXPORTER_OTLP_ENDPOINT: "http://{{ .Release.Name }}-otel-collector.{{ .Release.Namespace }}.svc.cluster.local:4318"
{{- end }}
159 changes: 159 additions & 0 deletions charts/glassflow-etl/templates/otlp-receiver-deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
{{- if .Values.sources.otlpReceiver.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}-otlp-receiver
namespace: {{ .Release.Namespace }}
labels:
{{- include "glassflow-etl.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.sources.otlpReceiver.replicas }}
selector:
matchLabels:
{{- include "glassflow-etl.selectorLabels" . | nindent 6 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 6 }}
{{- end }}
app.kubernetes.io/component: otlp-receiver
template:
metadata:
labels:
{{- include "glassflow-etl.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: otlp-receiver
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/otlp-receiver-config: {{ include (print $.Template.BasePath "/otlp-receiver-configmap.yaml") . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.affinity }}
affinity:
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
{{- if .Values.tolerations }}
tolerations:
{{- toYaml .Values.tolerations | nindent 8 }}
{{- end }}
initContainers:
{{- $natsAddress := .Values.global.nats.address | default (printf "nats://%s-nats.%s.svc.cluster.local:4222" .Release.Name .Release.Namespace) }}
- name: wait-for-nats
image: "busybox:1.35"
command:
- sh
- -c
- |
echo "Waiting for NATS to be ready..."
NATS_URL="{{ $natsAddress }}"
# Remove protocol prefix if present
NATS_HOST_PORT="${NATS_URL#nats://}"
NATS_HOST_PORT="${NATS_HOST_PORT#nats-secure://}"
# Extract host and port
if echo "$NATS_HOST_PORT" | grep -q ":"; then
NATS_HOST=$(echo "$NATS_HOST_PORT" | cut -d: -f1)
NATS_PORT=$(echo "$NATS_HOST_PORT" | cut -d: -f2)
else
NATS_HOST="$NATS_HOST_PORT"
NATS_PORT="4222"
fi
echo "Checking NATS at $NATS_HOST:$NATS_PORT..."
until nc -z "$NATS_HOST" "$NATS_PORT"; do
echo "NATS not ready, waiting..."
sleep 2
done
echo "NATS is ready!"
- name: wait-for-api
image: "busybox:1.35"
command:
- sh
- -c
- |
echo "Waiting for API to be ready..."
until nc -z {{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local {{ .Values.api.service.port }}; do
echo "API not ready, waiting..."
sleep 2
done
echo "API is ready!"
containers:
- name: otlp-receiver
image: "{{ .Values.global.imageRegistry | default "" }}{{ .Values.sources.otlpReceiver.image.repository }}:{{ .Values.sources.otlpReceiver.image.tag }}"
imagePullPolicy: {{ .Values.sources.otlpReceiver.image.pullPolicy }}
ports:
- name: grpc
containerPort: 4317
protocol: TCP
- name: http
containerPort: 4318
protocol: TCP
{{- with .Values.sources.otlpReceiver.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-otlp-receiver-config
env:
{{- if .Values.postgresql.enabled }}
- name: GLASSFLOW_DATABASE_URL
valueFrom:
secretKeyRef:
name: glassflow-postgresql
key: connection-url
{{- else if .Values.global.postgres.connection_url }}
- name: GLASSFLOW_DATABASE_URL
value: {{ .Values.global.postgres.connection_url | quote }}
{{- else if .Values.global.postgres.secret.name }}
- name: GLASSFLOW_DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ .Values.global.postgres.secret.name }}
key: {{ .Values.global.postgres.secret.key }}
{{- end }}
{{- if .Values.sources.otlpReceiver.env }}
{{- range .Values.sources.otlpReceiver.env }}
- name: {{ .name | quote }}
value: {{ .value | quote }}
{{- end }}
{{- end }}
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 3
readinessProbe:
httpGet:
path: /readyz
port: http
initialDelaySeconds: 3
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 2
{{- if and .Values.global.encryption.enabled .Values.global.encryption.existingSecret.name }}
volumeMounts:
- name: encryption-key
mountPath: /etc/glassflow/secrets
readOnly: true
{{- end }}
securityContext: {{- toYaml .Values.securityContext | nindent 12 }}
serviceAccountName: {{ include "glassflow-etl.serviceAccountName" . }}
securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- if and .Values.global.encryption.enabled .Values.global.encryption.existingSecret.name }}
volumes:
- name: encryption-key
secret:
secretName: {{ .Values.global.encryption.existingSecret.name }}
defaultMode: 0444
items:
- key: {{ .Values.global.encryption.existingSecret.key | default "encryption-key" }}
path: encryption-key
{{- end }}
{{- end }}
24 changes: 24 additions & 0 deletions charts/glassflow-etl/templates/otlp-receiver-service.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{{- if .Values.sources.otlpReceiver.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ .Release.Name }}-otlp-receiver
namespace: {{ .Release.Namespace }}
labels:
{{- include "glassflow-etl.labels" . | nindent 4 }}
app.kubernetes.io/component: otlp-receiver
spec:
type: {{ .Values.sources.otlpReceiver.service.type | default "ClusterIP" }}
ports:
- port: {{ .Values.sources.otlpReceiver.service.grpcPort | default 4317 }}
targetPort: grpc
protocol: TCP
name: grpc
- port: {{ .Values.sources.otlpReceiver.service.httpPort | default 4318 }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "glassflow-etl.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: otlp-receiver
{{- end }}
Loading
Loading