Upgrade Elasticsearch to 9.5.0

.github/workflows/build.yaml

@@ -83,6 +83,7 @@ jobs:
     timeout-minutes: 30
     outputs:
       version: ${{ steps.version.outputs.version }}
+      elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }}
       should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }}
       is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }}
       is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }}
@@ -129,9 +130,43 @@ jobs:
           echo "version=$version" >> $GITHUB_OUTPUT
           echo "### $version" >> $GITHUB_STEP_SUMMARY
 
+      - name: Resolve Elasticsearch image
+        id: elasticsearch_image
+        env:
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile)
+          tag=$version
+
+          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] &&
+            ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_sha=$(sha256sum build/docker/elasticsearch/9.x/Dockerfile | cut -d " " -f 1)
+            tag="$version-sha256-$image_sha"
+            image="exceptionless/elasticsearch:$tag"
+
+            for attempt in {1..30}; do
+              if docker manifest inspect "$image" > /dev/null 2>&1; then
+                break
+              fi
+
+              if [[ "$attempt" -eq 30 ]]; then
+                echo "::error::Timed out waiting for $image to be published."
+                exit 1
+              fi
+
+              sleep 10
+            done
+          fi
+
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+          echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY"
+
   test-api:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout
@@ -223,8 +258,11 @@ jobs:
         run: echo "npm run test:integration"
 
   test-e2e:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout

.github/workflows/elasticsearch-docker-8.yml

@@ -43,4 +43,4 @@ jobs:
         working-directory: build/docker/elasticsearch/8.x
         run: |
           VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile)
-          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION

.github/workflows/elasticsearch-docker-9.yml

@@ -0,0 +1,54 @@
+name: Elasticsearch 9.x Docker Image CI
+
+on:
+  push:
+    paths:
+      - "build/docker/elasticsearch/9.x/**"
+      - ".github/workflows/elasticsearch-docker-9.yml"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v') != true
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: 10.0.301
+      - name: Build Reason
+        env:
+          GITHUB_EVENT: ${{ toJson(github) }}
+        run: "echo ref: ${{github.ref}} event: ${{github.event_name}}"
+      - name: Build Version
+        run: |
+          dotnet tool install --global minver-cli --version 7.0.0
+          version=$(minver --tag-prefix v)
+          echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV
+          echo "VERSION=$version" >> $GITHUB_ENV
+          echo "### Version: $version" >> $GITHUB_STEP_SUMMARY
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+        with:
+          platforms: linux/amd64,linux/arm64
+      - name: Build custom Elasticsearch 9.x docker image
+        working-directory: build/docker/elasticsearch/9.x
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile)
+          if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then
+            TAGS=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest")
+          else
+            IMAGE_SHA=$(sha256sum Dockerfile | cut -d " " -f 1)
+            TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA")
+          fi
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . "${TAGS[@]}"

Dockerfile

@@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"]
 
 # completely self-contained
 
-FROM exceptionless/elasticsearch:8.19.15 AS exceptionless
+FROM exceptionless/elasticsearch:9.4.4 AS exceptionless
 
 WORKDIR /app
 COPY --from=job-publish /app/src/Exceptionless.Job/out ./

build/docker/elasticsearch/9.x/Dockerfile

@@ -0,0 +1,4 @@
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/elasticsearch/elasticsearch:9.4.4
+
+RUN elasticsearch-plugin install -b mapper-size

docker/docker-compose.apm.yml

@@ -2,7 +2,7 @@ version: "2.2"
 
 services:
   setup:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
     user: "0"
@@ -53,7 +53,7 @@ services:
     depends_on:
       setup:
         condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
       - esdata:/usr/share/elasticsearch/data
@@ -98,7 +98,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.4.4
     volumes:
       - certs:/usr/share/kibana/config/certs
     ports:
@@ -124,7 +124,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/apm/apm-server:8.19.15
+    image: docker.elastic.co/apm/apm-server:9.4.4
     volumes:
       - certs:/usr/share/apm-server/certs
     ports:

docker/docker-compose.dev.yml

@@ -50,7 +50,7 @@ services:
       - appdata:/app/storage
 
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.4.4
     environment:
       discovery.type: single-node
       xpack.security.enabled: "false"
@@ -74,7 +74,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.4.4
     ports:
       - 5601:5601
 

docker/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.4.4
     environment:
       node.name: elasticsearch
       cluster.name: exceptionless
@@ -26,7 +26,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.4.4
     environment:
       xpack.security.enabled: "false"
     ports:

k8s/elastic-monitor.yaml

@@ -4,7 +4,7 @@ metadata:
   name: elastic-monitor
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.4.4
   podDisruptionBudget: {}
   nodeSets:
     - name: main
@@ -228,7 +228,7 @@ metadata:
   name: kibana-monitor
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.4.4
   count: 1
   http:
     tls:
@@ -364,7 +364,7 @@ metadata:
   name: fleet-server
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.4.4
   kibanaRef:
     name: kibana-monitor
   elasticsearchRefs:
@@ -388,7 +388,7 @@ metadata:
   name: elastic-agent
   namespace: elastic-system
 spec:
-  version: 8.19.15
+  version: 9.4.4
   kibanaRef:
     name: kibana-monitor
   fleetServerRef:

k8s/ex-dev-elasticsearch.yaml

@@ -14,8 +14,8 @@ metadata:
   name: ex-dev
   namespace: ex-dev
 spec:
-  version: 8.19.15
-  image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
+  version: 9.4.4
+  image: exceptionless/elasticsearch:9.4.4 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
   secureSettings:
     - secretName: ex-dev-snapshots
   http:
@@ -68,7 +68,7 @@ metadata:
   name: ex-dev
   namespace: ex-dev
 spec:
-  version: 8.19.15
+  version: 9.4.4
   count: 1
   elasticsearchRef:
     name: ex-dev

k8s/ex-prod-elasticsearch.yaml

@@ -14,8 +14,8 @@ metadata:
   name: ex-prod
   namespace: ex-prod
 spec:
-  version: 8.19.15
-  image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
+  version: 9.4.4
+  image: exceptionless/elasticsearch:9.4.4 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
   monitoring:
     metrics:
       elasticsearchRefs:
@@ -79,7 +79,7 @@ metadata:
   name: ex-prod
   namespace: ex-prod
 spec:
-  version: 8.19.15
+  version: 9.4.4
   count: 1
   elasticsearchRef:
     name: ex-prod

k8s/exceptionless/values.yaml

@@ -35,7 +35,7 @@ elasticsearch:
   connectionString:
   image:
     repository: exceptionless/elasticsearch
-    tag: 8.19.15
+    tag: 9.4.4
     pullPolicy: IfNotPresent
 
 redis:

samples/docker-compose.all-in-one.yml

@@ -20,7 +20,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.4.4
     ports:
       - 5601:5601
 

samples/docker-compose.yml

@@ -44,7 +44,7 @@ services:
       - ex_appdata:/app/storage
 
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.4.4
     environment:
       discovery.type: single-node
       xpack.security.enabled: "false"
@@ -58,7 +58,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.4.4
     ports:
       - 5601:5601
 

src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs

@@ -13,7 +13,7 @@ public static class ElasticsearchBuilderExtensions
     private const int KibanaPort = 5601;
 
     /// <summary>
-    /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 8.19.15 tag of the Elasticsearch container image
+    /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.4.4 tag of the Elasticsearch container image
     /// </summary>
     /// <param name="builder">The <see cref="IDistributedApplicationBuilder"/>.</param>
     /// <param name="name">The name of the resource. This name will be used as the connection string name when referenced in a dependency.</param>
@@ -60,7 +60,11 @@ public static IResourceBuilder<ElasticsearchResource> AddElasticsearch(this IDis
             .PublishAsConnectionString();
     }
 
-    public static IResourceBuilder<ElasticsearchResource> WithKibana(this IResourceBuilder<ElasticsearchResource> builder, Action<IResourceBuilder<KibanaResource>>? configureContainer = null, string? containerName = null)
+    public static IResourceBuilder<ElasticsearchResource> WithKibana(
+        this IResourceBuilder<ElasticsearchResource> builder,
+        Action<IResourceBuilder<KibanaResource>>? configureContainer = null,
+        string? containerName = null,
+        int? port = null)
     {
         ArgumentNullException.ThrowIfNull(builder);
 
@@ -79,7 +83,7 @@ public static IResourceBuilder<ElasticsearchResource> WithKibana(this IResourceB
             var resourceBuilder = builder.ApplicationBuilder.AddResource(resource)
                                       .WithImage(ElasticsearchContainerImageTags.KibanaImage, ElasticsearchContainerImageTags.Tag)
                                       .WithImageRegistry(ElasticsearchContainerImageTags.KibanaRegistry)
-                                      .WithHttpEndpoint(targetPort: KibanaPort, name: containerName)
+                                      .WithHttpEndpoint(targetPort: KibanaPort, port: port, name: containerName)
                                       .WithUrlForEndpoint(containerName, u => u.DisplayText = "Kibana")
                                       .WithEnvironment("xpack.security.enabled", "false")
                                       .WithEnvironment(ctx =>
@@ -121,7 +125,7 @@ internal static class ElasticsearchContainerImageTags
     public const string Image = "exceptionless/elasticsearch";
     public const string KibanaRegistry = "docker.elastic.co";
     public const string KibanaImage = "kibana/kibana";
-    public const string Tag = "8.19.15";
+    public const string Tag = "9.4.4";
 }
 
 internal sealed class ElasticsearchConnectionHealthCheck(Func<string?> connectionStringFactory) : IHealthCheck
@@ -134,9 +138,17 @@ public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context
 
         using var settings = new ElasticsearchClientSettings(new Uri(connectionString));
         var client = new ElasticsearchClient(settings);
-        var response = await client.PingAsync(cancellationToken);
-        return response.IsValidResponse
-            ? HealthCheckResult.Healthy()
-            : new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch ping failed: {response.DebugInformation}");
+        var response = await client.Cluster.HealthAsync(
+            request => request.WaitForStatus(Elastic.Clients.Elasticsearch.HealthStatus.Yellow),
+            cancellationToken);
+        bool isReady = response.IsValidResponse
+            && !response.TimedOut
+            && response.Status is Elastic.Clients.Elasticsearch.HealthStatus.Yellow or Elastic.Clients.Elasticsearch.HealthStatus.Green;
+        if (isReady)
+            return HealthCheckResult.Healthy();
+
+        return new HealthCheckResult(
+            context.Registration.FailureStatus,
+            $"Elasticsearch cluster health check failed. Timed out: {response.TimedOut}; status: {response.Status}. {response.DebugInformation}");
     }
 }