chore: update Elasticsearch stack to 9.5.0

.github/workflows/build.yaml

@@ -83,6 +83,7 @@ jobs:
     timeout-minutes: 30
     outputs:
       version: ${{ steps.version.outputs.version }}
+      elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }}
       should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }}
       is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }}
       is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }}
@@ -129,9 +130,52 @@ jobs:
           echo "version=$version" >> $GITHUB_OUTPUT
           echo "### $version" >> $GITHUB_STEP_SUMMARY
 
+      - name: Resolve Elasticsearch image
+        id: elasticsearch_image
+        env:
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PUSH_BEFORE_SHA: ${{ github.event.before }}
+        run: |
+          version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile)
+          tag=$version
+          image_changed=false
+
+          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] &&
+            ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] &&
+            ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          fi
+
+          if [[ "$image_changed" == "true" ]]; then
+            image_sha=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1)
+            tag="$version-sha256-$image_sha"
+            image="exceptionless/elasticsearch:$tag"
+
+            for attempt in {1..150}; do
+              if docker manifest inspect "$image" > /dev/null 2>&1; then
+                break
+              fi
+
+              if [[ "$attempt" -eq 150 ]]; then
+                echo "::error::Timed out waiting for $image to be published."
+                exit 1
+              fi
+
+              sleep 10
+            done
+          fi
+
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+          echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY"
+
   test-api:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout
@@ -223,8 +267,11 @@ jobs:
         run: echo "npm run test:integration"
 
   test-e2e:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout
@@ -558,27 +605,6 @@ jobs:
           sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml
           helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-dev-values.yaml ex-dev --namespace $DEV_NAMESPACE ./k8s/exceptionless
 
-      - name: Run Development Migrations
-        if: ${{ needs.version.outputs.is_dev_deploy == 'true' }}
-        run: |
-          migration_job="ex-dev-jobs-migration-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          kubectl create job --from=cronjob/ex-dev-jobs-migration "$migration_job" --namespace "$DEV_NAMESPACE"
-
-          migration_image=$(kubectl get job "$migration_job" --namespace "$DEV_NAMESPACE" \
-            --output jsonpath='{.spec.template.spec.containers[0].image}')
-          if [[ "$migration_image" != *":${VERSION}" ]]; then
-            echo "::error::Migration job uses ${migration_image}; expected version ${VERSION}."
-            exit 1
-          fi
-
-          if ! kubectl wait --for=condition=complete --timeout=1200s "job/${migration_job}" --namespace "$DEV_NAMESPACE"; then
-            echo "::group::Migration job diagnostics"
-            kubectl describe job "$migration_job" --namespace "$DEV_NAMESPACE"
-            kubectl get pods --selector "job-name=${migration_job}" --namespace "$DEV_NAMESPACE" --output wide
-            echo "::endgroup::"
-            exit 1
-          fi
-
       - name: Ensure Development Workloads are Running
         if: ${{ needs.version.outputs.is_dev_deploy == 'true' }}
         run: |
@@ -599,6 +625,7 @@ jobs:
           kubectl patch cronjob/ex-dev-jobs-cleanup-orphaned-data -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
           kubectl patch cronjob/ex-dev-jobs-download-geoip-database -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
           kubectl patch cronjob/ex-dev-jobs-maintain-indexes -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
+          kubectl patch cronjob/ex-dev-jobs-migration -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
 
           wait_for_versioned_pod() {
             component="$1"

.github/workflows/elasticsearch-docker-8.yml

@@ -43,4 +43,4 @@ jobs:
         working-directory: build/docker/elasticsearch/8.x
         run: |
           VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile)
-          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION

.github/workflows/elasticsearch-docker-9.yml

@@ -0,0 +1,52 @@
+name: Elasticsearch 9.x Docker Image CI
+
+on:
+  push:
+    paths:
+      - "build/docker/elasticsearch/9.x/**"
+      - ".github/workflows/elasticsearch-docker-9.yml"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v') != true
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: 10.0.301
+      - name: Build Reason
+        env:
+          GITHUB_EVENT: ${{ toJson(github) }}
+        run: "echo ref: ${{github.ref}} event: ${{github.event_name}}"
+      - name: Build Version
+        run: |
+          dotnet tool install --global minver-cli --version 7.0.0
+          version=$(minver --tag-prefix v)
+          echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV
+          echo "VERSION=$version" >> $GITHUB_ENV
+          echo "### Version: $version" >> $GITHUB_STEP_SUMMARY
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+        with:
+          platforms: linux/amd64,linux/arm64
+      - name: Build custom Elasticsearch 9.x docker image
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile)
+          IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1)
+          TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA")
+          if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then
+            TAGS+=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest")
+          fi
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x "${TAGS[@]}"

Dockerfile

@@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"]
 
 # completely self-contained
 
-FROM exceptionless/elasticsearch:8.19.15 AS exceptionless
+FROM exceptionless/elasticsearch:9.5.0 AS exceptionless
 
 WORKDIR /app
 COPY --from=job-publish /app/src/Exceptionless.Job/out ./

build/docker/elasticsearch/9.x/Dockerfile

@@ -0,0 +1,4 @@
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.0
+
+RUN elasticsearch-plugin install -b mapper-size

docker/docker-compose.apm.yml

@@ -2,7 +2,7 @@ version: "2.2"
 
 services:
   setup:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
     user: "0"
@@ -53,7 +53,7 @@ services:
     depends_on:
       setup:
         condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
       - esdata:/usr/share/elasticsearch/data
@@ -98,7 +98,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.0
     volumes:
       - certs:/usr/share/kibana/config/certs
     ports:
@@ -124,7 +124,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/apm/apm-server:8.19.15
+    image: docker.elastic.co/apm/apm-server:9.5.0
     volumes:
       - certs:/usr/share/apm-server/certs
     ports:

docker/docker-compose.dev.yml

@@ -50,7 +50,7 @@ services:
       - appdata:/app/storage
 
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.5.0
     environment:
       discovery.type: single-node
       xpack.security.enabled: "false"
@@ -59,6 +59,8 @@ services:
       - 9200:9200
       - 9300:9300
     volumes:
+      # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9.
+      # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance
       - esdata7:/usr/share/elasticsearch/data
     healthcheck:
       test:
@@ -74,7 +76,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.0
     ports:
       - 5601:5601
 

docker/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.5.0
     environment:
       node.name: elasticsearch
       cluster.name: exceptionless
@@ -11,6 +11,8 @@ services:
     ports:
       - 9200:9200
     volumes:
+      # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9.
+      # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance
       - esdata:/usr/share/elasticsearch/data
     healthcheck:
       test:
@@ -26,7 +28,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.0
     environment:
       xpack.security.enabled: "false"
     ports:

docs/docs/self-hosting/upgrading-self-hosted-instance.md

@@ -8,6 +8,41 @@ title: "Upgrading"
 
 **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.**
 
+## Upgrading from v8 to v9
+
+Exceptionless v9 uses Elasticsearch 9. Do not point an Elasticsearch 9 node at an existing data volume until the cluster has been prepared with Elasticsearch 8.19. Elasticsearch 9 can fail to start when incompatible indices created before Elasticsearch 8 remain.
+
+Use this upgrade path for an existing self-hosted installation:
+
+1. Take a current Elasticsearch snapshot or other verified backup and test that it can be restored. Elasticsearch does not support downgrading a data directory after it has been upgraded.
+2. Upgrade Elasticsearch and Kibana to the latest 8.19.x patch release first, using the existing data volume. Do not start Elasticsearch 9 yet.
+3. Stop the Exceptionless app and job services, but leave Elasticsearch and Kibana 8.19 running. This prevents writes while legacy indices are reindexed.
+4. Open Kibana's **Upgrade Assistant** and resolve every critical issue. Reindex every active Exceptionless index created before Elasticsearch 8. Delete only indices you have confirmed are no longer needed; do not mark active Exceptionless indices as read-only.
+5. If this data volume previously ran Elasticsearch 7, temporarily disable the GeoIP downloader while still on Elasticsearch 8.19. Elasticsearch deletes its downloaded `.geoip_databases` system index when this setting is disabled; Exceptionless data is not affected.
+
+   ```bash
+   curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \
+     -H "Content-Type: application/json" \
+     -d '{"persistent":{"ingest.geoip.downloader.enabled":false}}'
+   ```
+
+6. Confirm that the deprecation API reports no critical issues:
+
+   ```bash
+   curl -fsS "http://localhost:9200/_migration/deprecations?pretty"
+   ```
+
+7. Stop Elasticsearch and Kibana 8.19 without deleting their data volume. Update the Elasticsearch and Kibana images to the v9 versions, start them, and verify cluster health before restarting the Exceptionless app and jobs. In Docker Compose, do not run `docker compose down -v` because `-v` deletes the data volume.
+8. After Elasticsearch 9 is healthy, restore the default GeoIP downloader behavior:
+
+   ```bash
+   curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \
+     -H "Content-Type: application/json" \
+     -d '{"persistent":{"ingest.geoip.downloader.enabled":null}}'
+   ```
+
+See Elastic's [prepare-to-upgrade guide](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade) for the supported 8.x to 9.x upgrade requirements and Upgrade Assistant details.
+
 ## Upgrading from v7.1 to v8
 
 We simplified the self hosting process by integrating the UI into the existing app images. As such `exceptionless/ui` docker images are deprecated and we recommend using `exceptionless/app`.

k8s/elastic-monitor.yaml

@@ -4,7 +4,7 @@ metadata:
   name: elastic-monitor
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.5.0
   podDisruptionBudget: {}
   nodeSets:
     - name: main
@@ -228,7 +228,7 @@ metadata:
   name: kibana-monitor
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.5.0
   count: 1
   http:
     tls:
@@ -364,7 +364,7 @@ metadata:
   name: fleet-server
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.5.0
   kibanaRef:
     name: kibana-monitor
   elasticsearchRefs:
@@ -388,7 +388,7 @@ metadata:
   name: elastic-agent
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.5.0
   kibanaRef:
     name: kibana-monitor
   fleetServerRef:

k8s/ex-dev-elasticsearch.yaml

@@ -14,8 +14,8 @@ metadata:
   name: ex-dev
   namespace: ex-dev
 spec:
-  version: 8.19.15
-  image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
+  version: 9.5.0
+  image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
   secureSettings:
     - secretName: ex-dev-snapshots
   http:
@@ -68,7 +68,7 @@ metadata:
   name: ex-dev
   namespace: ex-dev
 spec:
-  version: 8.19.15
+  version: 9.5.0
   count: 1
   elasticsearchRef:
     name: ex-dev

k8s/ex-prod-elasticsearch.yaml

@@ -14,8 +14,8 @@ metadata:
   name: ex-prod
   namespace: ex-prod
 spec:
-  version: 8.19.15
-  image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
+  version: 9.5.0
+  image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
   monitoring:
     metrics:
       elasticsearchRefs:
@@ -79,7 +79,7 @@ metadata:
   name: ex-prod
   namespace: ex-prod
 spec:
-  version: 8.19.15
+  version: 9.5.0
   count: 1
   elasticsearchRef:
     name: ex-prod

k8s/exceptionless/values.yaml

@@ -35,7 +35,7 @@ elasticsearch:
   connectionString:
   image:
     repository: exceptionless/elasticsearch
-    tag: 8.19.15
+    tag: 9.5.0
     pullPolicy: IfNotPresent
 
 redis: