Closes: #399 - Migrate away from bitnami/elasticsearch

.editorconfig

@@ -0,0 +1,5 @@
+root = true
+
+[*.sh]
+indent_style = space
+indent_size = 2

.github/ct.yaml

@@ -2,6 +2,7 @@
 chart-dirs: .
 chart-repos:
   - bitnami=https://charts.bitnami.com/bitnami
+  - elastic=https://helm.elastic.co
 check-version-increment: true
 debug: true
 # upgrade: true # Causes issues because of low performance on default GitHub actions servers.

.github/kubeconform.sh

@@ -13,10 +13,15 @@ curl --silent --show-error --fail --location --output /tmp/kubeconform.tar.gz ht
 sudo tar -C /usr/local/bin -xf /tmp/kubeconform.tar.gz kubeconform
 
 # validate charts
-for CHART_DIR in ${CHART_DIRS};do
+for CHART_DIR in ${CHART_DIRS}; do
   echo "helm dependency build…"
   helm dependency build "${CHART_DIR}"
 
   echo "kubeconform(ing) ${CHART_DIR##charts/} chart…"
-  helm template "${CHART_DIR}" | kubeconform --strict --verbose --kubernetes-version "${KUBERNETES_VERSION#v}"
+  # The second schema location resolves CRD-based kinds (e.g. the ECK-managed
+  # Elasticsearch resource) from the datree CRDs catalog.
+  helm template "${CHART_DIR}" | kubeconform --strict --verbose \
+    --schema-location default \
+    --schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
+    --kubernetes-version "${KUBERNETES_VERSION#v}"
 done

.github/updatecli.yaml

@@ -18,14 +18,21 @@ sources:
       versionfilter:
         pattern: '3.x.x'
         kind: semver
-  # elasticsearch:
-  #   name: elasticsearch
-  #   kind: helmchart
-  #   spec:
-  #     url: https://charts.bitnami.com/bitnami
-  #     name: elasticsearch
-  #     versionfilter:
-  #       pattern: "21.x.x"
+  elasticsearch:
+    name: elasticsearch
+    kind: helmchart
+    spec:
+      url: https://helm.elastic.co
+      name: eck-elasticsearch
+      # Patch level only: this subchart provides the Elasticsearch version (which we do not
+      #   pin, see values.yaml) and it may change the Elasticsearch minor or major version in
+      #   a chart minor release, as the chart itself stays on 0.x. Chart patches only ever
+      #   carry Elasticsearch patch updates.
+      #   To perform a minor update, raise the pattern below and check that the ECK operator
+      #   installed in the cluster (a prerequisite outside of this chart, see README) supports
+      #   the resulting Elasticsearch version.
+      versionfilter:
+        pattern: "0.20.x"
   # minio:
   #   name: minio
   #   kind: helmchart
@@ -102,18 +109,18 @@ targets:
       file: zammad/values.yaml
       matchpattern: '        tag: "\d+\.\d+\.\d+"'
       replacepattern: '        tag: "{{ source `alpine` }}"'
-  # elasticsearch:
-  #   name: elasticsearch
-  #   kind: helmchart
-  #   sourceid: elasticsearch
-  #   # {{ if .github.enabled }}
-  #   scmid: zammad-helm
-  #   # {{ end }}
-  #   spec:
-  #     name: "zammad"
-  #     file: "Chart.yaml"
-  #     key: "$.dependencies[0].version"
-  #     versionincrement: patch
+  elasticsearch:
+    name: elasticsearch
+    kind: helmchart
+    sourceid: elasticsearch
+    # {{ if .github.enabled }}
+    scmid: zammad-helm
+    # {{ end }}
+    spec:
+      name: "zammad"
+      file: "Chart.yaml"
+      key: "$.dependencies[0].version"
+      versionincrement: patch
   # minio:
   #   name: minio
   #   kind: helmchart

.github/workflows/ci.yaml

@@ -123,6 +123,14 @@ jobs:
           config: .github/kind-config.yaml
           node_image: kindest/node:${{ matrix.k8s }}
 
+      - name: Install ECK operator (CRDs + operator)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          helm repo add elastic https://helm.elastic.co
+          helm repo update
+          helm install elastic-operator elastic/eck-operator \
+            --namespace elastic-system --create-namespace --wait
+
       - name: Create Namespace 'zammad'
         run: kubectl create namespace zammad
 

zammad/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: zammad
-version: 17.0.1
+version: 18.0.0
 appVersion: 7.1.2-0004
 description: Zammad is a web based open source helpdesk/customer support system with many features to manage customer communication via several channels like telephone, facebook, twitter, chat and e-mails.
 home: https://zammad.org
@@ -14,9 +14,14 @@ maintainers:
   - name: mgruner
     email: enjoy@zammad.com
 dependencies:
-  - name: elasticsearch
-    repository: https://charts.bitnami.com/bitnami
-    version: 21.6.3
+  # Official Elastic way to run Elasticsearch: the eck-elasticsearch chart deploys
+  # an Elasticsearch custom resource that is reconciled by the ECK operator.
+  # The ECK operator (+ CRDs) must be installed cluster-wide as a prerequisite,
+  # see README. Aliased to "elasticsearch" to keep the values key stable.
+  - name: eck-elasticsearch
+    alias: elasticsearch
+    repository: https://helm.elastic.co
+    version: 0.20.0
     condition: zammadConfig.elasticsearch.enabled
   - name: minio
     version: 14.10.5

zammad/README.md

@@ -41,9 +41,26 @@ Be aware that the Zammad Helm chart version is different from the actual Zammad
 
 ## Prerequisites
 
-- Kubernetes 1.19+
+- Kubernetes 1.21+
 - Helm 3.8.0+
 - Cluster with at least 4GB of free RAM
+- The [ECK operator](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-install-helm.html),
+  **version 3.5.0 or newer**, must be installed in the cluster when using the bundled
+  Elasticsearch (`zammadConfig.elasticsearch.enabled: true`, the default). The bundled
+  Elasticsearch is deployed via the official `eck-elasticsearch` chart (pinned to `0.20.0`
+  in `Chart.yaml`), which creates an `Elasticsearch` custom resource that is reconciled by
+  the operator. If your cluster already runs an
+  older ECK operator for other workloads, upgrade it first — an older operator's webhook may
+  reject the `Elasticsearch` resource this chart creates. This is **not** required when
+  connecting to an external Elasticsearch service.
+
+  Install the operator once per cluster:
+
+  ```console
+  helm repo add elastic https://helm.elastic.co
+  helm repo update
+  helm install elastic-operator elastic/eck-operator --namespace elastic-system --create-namespace
+  ```
 
 ## Installing the Chart
 
@@ -114,7 +131,11 @@ To deploy on OpenShift unprivileged and with [arbitrary UIDs and GIDs](https://c
 - [Delete the default key](https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key) `securityContext` and `zammadConfig.initContainers.zammad.securityContext.runAsUser` with `null`.
 - Disable if used:
   - also `podSecurityContext` in all subcharts.
-  - the privileged [sysctlImage](https://github.com/bitnami/charts/tree/main/bitnami/elasticsearch#default-kernel-settings) in elasticsearch subchart.
+
+The bundled Elasticsearch uses `node.store.allow_mmap: false` by default (see
+`elasticsearch.nodeSets[].config`), so no privileged sysctl init container is
+required. If you raise `vm.max_map_count` for production instead, do so on the
+node level rather than via a privileged init container.
 
 ```yaml
 securityContext: null
@@ -130,15 +151,6 @@ zammadConfig:
     emptyDir:
       medium: memory
 
-elasticsearch:
-  sysctlImage:
-    enabled: false
-  master:
-    podSecurityContext:
-      enabled: false
-    containerSecurityContext:
-      enabled: false
-
 memcached:
   podSecurityContext:
     enabled: false
@@ -167,19 +179,15 @@ redis:
 ### Deploying with ArgoCD
 
 Due to the way Argo CD syncs Helm charts into the cluster and this chart deploying the initialization job, the default configuration can lead to Sync loops where Argo CD will create an infinite amount of initialization jobs.
-To prevent this, disable the random name for the initialization job and add the according annotation to the job to let Argo CD treat it as a Sync Hook.
+To prevent this, add the according annotation to the job to let Argo CD treat it as a Sync Hook.
 
 ```yaml
 zammadConfig:
   initJob:
-    randomName: false
     annotations:
       argocd.argoproj.io/hook: Sync
 ```
 
-Note that `randomName` is a bit of a misnomer: the job name suffix is the Helm release revision, not a random value.
-Tools that render the chart statically instead of performing a real Helm install/upgrade (Argo CD, `helm template`, `helm diff`) always see revision `1`, so with `randomName: true` the job name never changes across syncs and the job will not re-run on its own - hence the `false` + Sync Hook workaround above. Flux's helm-controller uses the Helm SDK directly and is unaffected by this.
-
 ## Maintenance Tasks
 
 If you have to run any maintenance commands inside of the Zammad stack, you can do that by
@@ -201,6 +209,74 @@ and `zammadConfig.cronJob.reindex.schedule` if you want to run it periodically.
 
 ## Upgrading
 
+### From Chart Version 17.x to 18.0.0
+
+#### Elasticsearch subchart migrated to the ECK operator
+
+- The bundled Elasticsearch was migrated from the `bitnami/elasticsearch` subchart to the
+  official [`eck-elasticsearch`](https://artifacthub.io/packages/helm/elastic/eck-elasticsearch)
+  chart, which is managed by the [ECK operator](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-overview.html).
+  This is the deployment method officially supported by Elastic.
+- **The ECK operator (and its CRDs), version 3.5.0+, must now be installed in the cluster**
+  before upgrading when `zammadConfig.elasticsearch.enabled` is `true` (the default). See
+  [Prerequisites](#prerequisites).
+- The `elasticsearch.*` values changed completely. The previous bitnami values (`master`, `data`,
+  `ingest`, `coordinating`, `sysctlImage`, `image`, `global.security.allowInsecureImages`, …) no
+  longer apply. Review the new `elasticsearch.*` block in `values.yaml` (`nodeSets`, `http`, …).
+- Authentication is now mandatory: ECK creates a superuser `elastic` and stores its password in
+  an auto-generated secret, typically named `{{ .Release.Name }}-es-es-elastic-user` (depending
+  on your release name, see below). The chart wires this up automatically, so
+  `secrets.elasticsearch.*` is only relevant for an **external** Elasticsearch now. Since TLS on
+  the HTTP layer is disabled by default (see below), this password still crosses the cluster
+  network in plaintext on every request, same as the previous unauthenticated setup did for all
+  traffic — authentication alone does not protect it in transit.
+- The in-cluster service name changed from `{{ .Release.Name }}-elasticsearch` to typically
+  `{{ .Release.Name }}-es-es-http`. The exact name depends on your release name: if it already
+  contains `es` as a substring (e.g. `zammad-helpdesk`), it's used as-is instead of getting `-es`
+  appended, giving `{{ .Release.Name }}-es-http` instead. Either way, the chart's own templates
+  resolve this automatically. Note that ECK caps the Elasticsearch resource name itself at 36
+  characters; the chart fails fast with a clear error at template/install time if your release
+  name is too long to fit.
+- TLS on the HTTP layer is disabled by default (`elasticsearch.http.tls.selfSignedCertificate.disabled: true`)
+  to keep the plain-`http` connection behaviour. Enabling the operator-managed self-signed
+  certificate (`selfSignedCertificate.disabled: false` with `zammadConfig.elasticsearch.schema: https`)
+  is **not supported yet**: Zammad verifies TLS certificates by default and the chart does not mount
+  the ECK-generated CA certificate into the Zammad pods, so the zammad-init job will fail on
+  certificate verification. Leave TLS disabled unless you terminate it yourself and configure
+  Zammad's Elasticsearch client accordingly.
+- The StatefulSet and its `PersistentVolumeClaim`s are recreated under new names. As the search
+  index can be rebuilt from PostgreSQL, the recommended path is to delete the old bitnami
+  Elasticsearch PVCs and let the index be recreated. This usually happens automatically when the
+  zammad-init job runs and finds no index to be available.
+- **Behaviour change:** the ECK operator's default `volumeClaimDeletePolicy` is
+  `DeleteOnScaledownAndClusterDeletion`, so uninstalling the release (or setting
+  `zammadConfig.elasticsearch.enabled: false`) now also deletes the Elasticsearch data volume.
+  The old bitnami PVCs were left behind on uninstall. This is harmless in practice since the
+  index rebuilds from PostgreSQL, but if you rely on the volume surviving an uninstall, set
+  `elasticsearch.volumeClaimDeletePolicy: DeleteOnScaledownOnly` explicitly.
+
+#### Faster volume permission fixing
+
+- `securityContext.fsGroupChangePolicy` now defaults to `OnRootMismatch` instead of `Always`. This
+  avoids a lengthy startup delay on volumes with a large number of files, since Kubernetes only
+  recurses into a volume to fix ownership if the volume's root directory doesn't already have the
+  expected group ownership. This is safe for existing volumes (their ownership was already set by
+  a previous `Always` run) and only changes behaviour on future pod restarts. If you rely on the
+  old, more thorough behaviour, set `securityContext.fsGroupChangePolicy: Always` explicitly.
+
+#### `initJob.randomName` removed
+
+- The `zammadConfig.initJob.randomName` setting was removed. The init Job name is now always
+  suffixed with the Helm release revision, which is what `randomName: true` (the default) already
+  did in practice.
+- If you had set `randomName: false`, e.g. for the Argo CD workaround described in
+  [Deploying with ArgoCD](#deploying-with-argocd), remove that line - it is now a no-op. Argo CD
+  and other tools that render the chart statically (also `helm template`, `helm diff`) always see
+  revision `1`, so the job name is stable there regardless. It's the Sync Hook annotation that
+  actually prevents the sync loop, and that is unaffected by this change.
+- On upgrade, installs that previously had `randomName: false` will see their init Job renamed once
+  (`<fullname>-init` to `<fullname>-init-<revision>`); this is a harmless one-time replacement.
+
 ### From Chart Version 16.x to 17.0.0
 
 - The bitnami PostgreSQL subchart was replaced with [cloudpirates-postgres/postgres](https://artifacthub.io/packages/helm/cloudpirates-postgres/postgres).

zammad/ci/full-values.yaml

@@ -24,8 +24,7 @@ autoWizard:
   enabled: true
 
 elasticsearch:
-  security:
-    existingSecret: elasticsearch-existing-secret
+  nameOverride: my-local-es
 
 ingress:
   enabled: true

zammad/templates/_helpers.tpl

@@ -105,17 +105,78 @@ autowizard secret name
 {{- end -}}
 {{- end -}}
 
+{{/*
+Name of the ECK Elasticsearch resource. Mirrors the eck-elasticsearch chart's
+"elasticsearch.fullname" logic so that ECK-derived names (services, secrets)
+stay correct even if elasticsearch.nameOverride/fullnameOverride are changed.
+Also fails fast if the result exceeds ECK's 36-character resource name limit,
+since the admission webhook's own rejection of an over-long name is opaque.
+*/}}
+{{- define "zammad.elasticsearchName" -}}
+{{- $es := .Values.elasticsearch | default dict -}}
+{{- $name := "" -}}
+{{- if $es.fullnameOverride -}}
+{{- $name = $es.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $base := default "elasticsearch" $es.nameOverride -}}
+{{- if contains $base .Release.Name -}}
+{{- $name = .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name = printf "%s-%s" .Release.Name $base | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- if gt (len $name) 36 -}}
+{{- fail (printf "Elasticsearch resource name %q is %d characters, but the ECK operator caps names at 36. Shorten the release name or set a shorter elasticsearch.nameOverride/fullnameOverride." $name (len $name)) -}}
+{{- end -}}
+{{- $name -}}
+{{- end -}}
+
+{{/*
+elasticsearch service host (in-cluster ECK service or external host)
+*/}}
+{{- define "zammad.elasticsearchHost" -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+{{ include "zammad.elasticsearchName" . }}-es-http
+{{- else -}}
+{{ .Values.zammadConfig.elasticsearch.host }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+elasticsearch user (ECK creates the "elastic" superuser for the in-cluster service)
+*/}}
+{{- define "zammad.elasticsearchUser" -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+elastic
+{{- else -}}
+{{ .Values.zammadConfig.elasticsearch.user }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 elasticsearch secret name
 */}}
 {{- define "zammad.elasticsearchSecretName" -}}
-{{- if .Values.secrets.elasticsearch.useExisting -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+{{ include "zammad.elasticsearchName" . }}-es-elastic-user
+{{- else if .Values.secrets.elasticsearch.useExisting -}}
 {{ .Values.secrets.elasticsearch.secretName }}
 {{- else -}}
 {{ include "zammad.fullname" . }}-{{ .Values.secrets.elasticsearch.secretName }}
 {{- end -}}
 {{- end -}}
 
+{{/*
+elasticsearch secret key (ECK stores the password under the user name "elastic")
+*/}}
+{{- define "zammad.elasticsearchSecretKey" -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+elastic
+{{- else -}}
+{{ .Values.secrets.elasticsearch.secretKey }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 postgresql secret name
 */}}

zammad/templates/configmap-init.yaml

@@ -45,10 +45,10 @@ data:
     #!/bin/bash
     set -e
 
-    ELASTICSEARCH_URL={{ .Values.zammadConfig.elasticsearch.schema }}://{{ if .Values.zammadConfig.elasticsearch.enabled }}{{ .Release.Name }}-elasticsearch{{ else }}{{ .Values.zammadConfig.elasticsearch.host }}{{ end }}:{{ .Values.zammadConfig.elasticsearch.port }}
+    ELASTICSEARCH_URL={{ .Values.zammadConfig.elasticsearch.schema }}://{{ include "zammad.elasticsearchHost" . }}:{{ .Values.zammadConfig.elasticsearch.port }}
     bundle exec rails r "Setting.set('es_url', '${ELASTICSEARCH_URL}')"
 
-    ELASTICSEARCH_USER=${ELASTICSEARCH_USER:-{{ .Values.zammadConfig.elasticsearch.user }}}
+    ELASTICSEARCH_USER=${ELASTICSEARCH_USER:-{{ include "zammad.elasticsearchUser" . }}}
     if [ -n "${ELASTICSEARCH_USER}" ] && [ -n "${ELASTICSEARCH_PASSWORD}" ]; then
         bundle exec rails r "Setting.set('es_user', '${ELASTICSEARCH_USER}'); Setting.set('es_password', '${ELASTICSEARCH_PASSWORD}')"
     fi