[ENG-3846]: Add liveness probe for temporal workers

charts/datafold-manager/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: datafold-manager
 description: Helm chart for Datafold Operator
 type: application
-version: 0.1.103
+version: 0.1.104
 appVersion: "1.0.0"
 icon: https://www.datafold.com/logo.png
 

charts/datafold-manager/values.yaml

@@ -18,7 +18,7 @@ operator:
   # Operator image configuration
   image:
     repository: us-docker.pkg.dev/datadiff-mm/datafold/datafold-operator
-    tag: "1.1.71"
+    tag: "1.1.72"
     pullPolicy: Always
 
   # Operator deployment configuration

charts/datafold/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: datafold
 description: Helm chart package to deploy Datafold on kubernetes.
 type: application
-version: 0.10.87
+version: 0.10.88
 appVersion: "1.0.0"
 icon: https://www.datafold.com/logo.png
 

charts/datafold/charts/worker-temporal/templates/deployment.yaml

@@ -103,14 +103,35 @@ spec:
             - name: TEMPORAL_METRICS_BIND_ADDRESS
               value: "0.0.0.0:{{ .Values.metrics.port }}"
             {{- end }}
+            {{- if .Values.liveness.enabled }}
+            - name: TEMPORAL_LIVENESS_PORT
+              value: {{ .Values.liveness.port | quote }}
+            {{- end }}
             {{- with .Values.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
-          {{- if .Values.metrics.enabled }}
+          {{- if or .Values.metrics.enabled .Values.liveness.enabled }}
           ports:
+            {{- if .Values.metrics.enabled }}
             - name: metrics
               containerPort: {{ .Values.metrics.port }}
               protocol: TCP
+            {{- end }}
+            {{- if .Values.liveness.enabled }}
+            - name: liveness
+              containerPort: {{ .Values.liveness.port }}
+              protocol: TCP
+            {{- end }}
+          {{- end }}
+          {{- if .Values.liveness.enabled }}
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: {{ .Values.liveness.port }}
+            initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
+            periodSeconds: {{ .Values.liveness.periodSeconds }}
+            failureThreshold: {{ .Values.liveness.failureThreshold }}
+            timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
           {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}

charts/datafold/charts/worker-temporal/values.yaml

@@ -76,6 +76,17 @@ metrics:
   enabled: true
   port: 9090
 
+liveness:
+  # Starts a lightweight HTTP server inside the worker and wires up a K8s
+  # livenessProbe against /livez. The probe returns 503 when any Temporal worker
+  # stops running, a fatal error is raised, or the process pool becomes broken.
+  enabled: true
+  port: 8091
+  initialDelaySeconds: 60
+  periodSeconds: 20
+  failureThreshold: 3
+  timeoutSeconds: 5
+
 extraEnv: []
 
 volumes: []