diff --git a/charts/glassflow-etl/Chart.yaml b/charts/glassflow-etl/Chart.yaml index 6ae1683..2efc357 100644 --- a/charts/glassflow-etl/Chart.yaml +++ b/charts/glassflow-etl/Chart.yaml @@ -15,17 +15,17 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.16 +version: 0.5.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "3.0.0" +appVersion: "3.1.0" dependencies: - name: glassflow-operator - version: "0.8.0" + version: "0.8.1" repository: https://glassflow.github.io/glassflow-etl-k8s-operator - name: nats version: "2.12.3" diff --git a/charts/glassflow-etl/templates/deployment.yaml b/charts/glassflow-etl/templates/deployment.yaml index aeb2617..6fdd114 100644 --- a/charts/glassflow-etl/templates/deployment.yaml +++ b/charts/glassflow-etl/templates/deployment.yaml @@ -258,6 +258,46 @@ spec: memory: "512Mi" cpu: "500m" securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + - name: run-data-migration + image: "{{ .Values.global.imageRegistry | default "" }}{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}" + imagePullPolicy: {{ .Values.api.image.pullPolicy }} + args: ["-role=migrate-data"] + env: + {{- if .Values.postgresql.enabled }} + - name: GLASSFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: glassflow-postgresql + {{- if .Values.global.pgbouncer.enabled }} + key: direct-connection-url + {{- else }} + key: connection-url + {{- end }} + {{- else if .Values.global.postgres.direct_connection_url }} + - name: GLASSFLOW_DATABASE_URL + value: {{ .Values.global.postgres.direct_connection_url | quote }} + {{- else if .Values.global.postgres.connection_url }} + - name: GLASSFLOW_DATABASE_URL + value: {{ .Values.global.postgres.connection_url | quote }} + {{- else if .Values.global.postgres.secret.name }} + - name: GLASSFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.global.postgres.secret.name }} + {{- if .Values.global.postgres.secret.direct_key }} + key: {{ .Values.global.postgres.secret.direct_key }} + {{- else }} + key: {{ .Values.global.postgres.secret.key }} + {{- end }} + {{- end }} + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + securityContext: {{- toYaml .Values.securityContext | nindent 12 }} {{- end }} containers: - name: api diff --git a/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml index 382e74d..f17f019 100644 --- a/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml +++ b/charts/glassflow-etl/templates/otlp-receiver-configmap.yaml @@ -11,6 +11,10 @@ data: GLASSFLOW_ROLE: "otlp-receiver" GLASSFLOW_NATS_SERVER: "{{ .Values.global.nats.address | default (printf "nats://%s-nats.%s.svc.cluster.local:4222" .Release.Name .Release.Namespace) }}" GLASSFLOW_OTLP_CONFIG_FETCHER_BASE_URL: "http://{{ .Release.Name }}-api.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.api.service.port }}" + GLASSFLOW_OTLP_MAX_CONCURRENT_REQUESTS: "{{ .Values.sources.otlpReceiver.maxConcurrentRequests }}" + GLASSFLOW_OTLP_NATS_CHUNK_SIZE: "{{ .Values.sources.otlpReceiver.natsChunkSize }}" + GLASSFLOW_OTEL_METRICS_ENABLED: "{{ .Values.global.observability.metrics.enabled }}" + GLASSFLOW_OTEL_LOGS_ENABLED: "{{ .Values.global.observability.logs.enabled }}" GLASSFLOW_LOG_LEVEL: "info" GLASSFLOW_RUN_LOCAL: "false" OTEL_EXPORTER_OTLP_ENDPOINT: "http://{{ .Release.Name }}-otel-collector.{{ .Release.Namespace }}.svc.cluster.local:4318" diff --git a/charts/glassflow-etl/values.yaml b/charts/glassflow-etl/values.yaml index 2f43906..32fff45 100644 --- a/charts/glassflow-etl/values.yaml +++ b/charts/glassflow-etl/values.yaml @@ -152,7 +152,7 @@ api: logLevel: "info" image: repository: glassflow-etl-be - tag: v3.0.0 + tag: v3.1.0 pullPolicy: IfNotPresent resources: requests: @@ -187,7 +187,7 @@ ui: replicas: 1 image: repository: glassflow-etl-fe - tag: v3.0.0 + tag: v3.1.0 pullPolicy: IfNotPresent logLevel: "info" resources: @@ -266,7 +266,7 @@ glassflow-operator: reconcileTimeout: 15m image: repository: glassflow-etl-k8s-operator - tag: v3.0.0 + tag: v3.1.0 pullPolicy: IfNotPresent resources: limits: @@ -282,7 +282,7 @@ glassflow-operator: ingestor: image: repository: glassflow-etl-ingestor - tag: v3.0.0 + tag: v3.1.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -297,7 +297,7 @@ glassflow-operator: join: image: repository: glassflow-etl-join - tag: v3.0.0 + tag: v3.1.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -312,7 +312,7 @@ glassflow-operator: sink: image: repository: glassflow-etl-sink - tag: v3.0.0 + tag: v3.1.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -327,7 +327,7 @@ glassflow-operator: dedup: image: repository: glassflow-etl-dedup - tag: v3.0.0 + tag: v3.1.0 pullPolicy: IfNotPresent logLevel: "INFO" resources: @@ -353,8 +353,8 @@ sources: replicas: 1 image: repository: glassflow-etl-otlp-receiver - tag: v3.0.0 - pullPolicy: Always + tag: v3.1.0 + pullPolicy: IfNotPresent resources: requests: memory: "256Mi" @@ -366,7 +366,14 @@ sources: type: ClusterIP grpcPort: 4317 httpPort: 4318 - # Environment variables for the OTLP receiver + # Max concurrent in-flight batches before the receiver starts shedding load + # (returns HTTP 503 / gRPC ResourceExhausted). Increase if you need higher + # throughput but ensure the pod has enough CPU/memory headroom. + maxConcurrentRequests: 50 + # Max messages per NATS async-publish chunk. Keeps per-request memory bounded + # regardless of upstream batch size. + natsChunkSize: 1000 + # Additional environment variables for the OTLP receiver # Example: # env: # - name: GLASSFLOW_LOG_LEVEL