diff --git a/lab/iam/policies/labs1.yaml b/lab/iam/policies/labs1.yaml index 2326a83338..065e01c5c0 100644 --- a/lab/iam/policies/labs1.yaml +++ b/lab/iam/policies/labs1.yaml @@ -135,6 +135,11 @@ Statement: - Effect: Allow Action: - cloudwatch:GetMetricData + - cloudwatch:StartOTelEnrichment + Resource: ["*"] + - Effect: Allow + Action: + - observabilityadmin:StartTelemetryEnrichment Resource: ["*"] - Effect: Allow Action: diff --git a/manifests/modules/observability/otlp-metrics/.workshop/cleanup.sh b/manifests/modules/observability/otlp-metrics/.workshop/cleanup.sh new file mode 100644 index 0000000000..292bf9c9f4 --- /dev/null +++ b/manifests/modules/observability/otlp-metrics/.workshop/cleanup.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +logmessage "Cleaning up OTLP metrics resources..." + +kubectl delete amazoncloudwatchagent application-metrics-collector -n amazon-cloudwatch --ignore-not-found +kubectl delete -n other pod load-generator --ignore-not-found + +logmessage "Disabling OTel enrichment..." +aws cloudwatch stop-otel-enrichment 2>/dev/null || true + +logmessage "Disabling resource tags on telemetry..." +aws observabilityadmin stop-telemetry-enrichment 2>/dev/null || true diff --git a/manifests/modules/observability/otlp-metrics/.workshop/terraform/main.tf b/manifests/modules/observability/otlp-metrics/.workshop/terraform/main.tf new file mode 100644 index 0000000000..7fa317c65f --- /dev/null +++ b/manifests/modules/observability/otlp-metrics/.workshop/terraform/main.tf @@ -0,0 +1,359 @@ +data "aws_partition" "current" {} +data "aws_region" "current" {} + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "1.23.0" + + cluster_name = var.addon_context.eks_cluster_id + cluster_endpoint = var.addon_context.aws_eks_cluster_endpoint + cluster_version = var.eks_cluster_version + oidc_provider_arn = var.addon_context.eks_oidc_provider_arn + + enable_aws_load_balancer_controller = true + aws_load_balancer_controller = { + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" + } + + observability_tag = null +} + +resource "aws_eks_addon" "pod_identity" { + cluster_name = var.addon_context.eks_cluster_id + addon_name = "eks-pod-identity-agent" + resolve_conflicts_on_create = "OVERWRITE" + preserve = false +} + +# IAM role for the CloudWatch Observability add-on (Pod Identity) +resource "aws_iam_role" "cloudwatch_observability" { + name = "${var.addon_context.eks_cluster_id}-cw-observability" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { + Service = "pods.eks.amazonaws.com" + } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "cloudwatch_observability" { + role = aws_iam_role.cloudwatch_observability.name + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/CloudWatchAgentServerPolicy" +} + +resource "aws_eks_pod_identity_association" "cloudwatch_observability" { + cluster_name = var.addon_context.eks_cluster_id + namespace = "amazon-cloudwatch" + service_account = "cloudwatch-agent" + role_arn = aws_iam_role.cloudwatch_observability.arn + + depends_on = [aws_eks_addon.pod_identity] +} + +resource "aws_eks_addon" "amazon_cloudwatch_observability" { + cluster_name = var.addon_context.eks_cluster_id + addon_name = "amazon-cloudwatch-observability" + resolve_conflicts_on_create = "OVERWRITE" + + configuration_values = jsonencode({ + otelContainerInsights = { + enabled = true + logs = { + enabled = false + } + } + containerLogs = { + enabled = false + } + applicationSignals = { + enabled = false + } + }) + + depends_on = [aws_eks_pod_identity_association.cloudwatch_observability] +} + +# Enable resource tags on telemetry (account-level prerequisite for OTel enrichment) +resource "terraform_data" "telemetry_enrichment" { + provisioner "local-exec" { + command = "aws observabilityadmin start-telemetry-enrichment" + } +} + +# Enable OTel enrichment for AWS vended metrics (makes them queryable via PromQL with resource tags) +resource "terraform_data" "otel_enrichment" { + depends_on = [terraform_data.telemetry_enrichment] + + provisioner "local-exec" { + command = "aws cloudwatch start-otel-enrichment" + } +} + +resource "time_sleep" "blueprints_addons_sleep" { + depends_on = [ + module.eks_blueprints_addons, + ] + + create_duration = "15s" +} + +# Grafana setup with CloudWatch OTLP metrics as Prometheus-compatible data source + +resource "kubernetes_namespace" "grafana" { + metadata { + name = "grafana" + } +} + +module "grafana" { + source = "aws-ia/eks-blueprints-addon/aws" + version = "1.1.1" + + depends_on = [ + time_sleep.blueprints_addons_sleep, + kubernetes_config_map.order_service_metrics_dashboard + ] + + description = "Grafana" + chart = "grafana" + chart_version = var.grafana_chart_version + namespace = kubernetes_namespace.grafana.metadata[0].name + create_namespace = false + repository = "https://grafana.github.io/helm-charts" + values = [local.grafana_values] + wait = true + set = [{ + name = "serviceAccount.name" + value = "grafana" + }] + + create_role = true + role_name = "${var.addon_context.eks_cluster_id}-grafana" + policy_name = "${var.addon_context.eks_cluster_id}-grafana" + source_policy_documents = [data.aws_iam_policy_document.grafana.json] + set_irsa_names = [ + "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn", + ] + oidc_providers = { + this = { + provider_arn = var.addon_context.eks_oidc_provider_arn + service_account = "grafana" + } + } +} + +resource "kubernetes_config_map" "order_service_metrics_dashboard" { + metadata { + name = "order-service-metrics-dashboard" + namespace = kubernetes_namespace.grafana.metadata[0].name + + labels = { + grafana_dashboard = 1 + } + } + + data = { + "order-service-metrics-dashboard.json" = <.amazonaws.com` +4. Set **Service Provider** to `monitoring` +5. Set **Region** to your AWS Region +6. Choose **Default Credential Chain** as authentication provider +7. Click **Save & test** + +Refer to the official documentation: [CloudWatch PromQL](https://docs.aws.amazon.com/grafana/latest/userguide/cloudwatch-promql.html) + +## Files Modified + +| File | Change | +|------|--------| +| `manifests/modules/observability/otlp-metrics/.workshop/terraform/main.tf` | Added Pod Identity agent, IAM role, Pod Identity association, CloudWatch observability add-on (OTLP metrics only), telemetry enrichment, OTel enrichment | +| `manifests/modules/observability/otlp-metrics/.workshop/cleanup.sh` | Added cleanup for OTel enrichment and telemetry enrichment | +| `lab/iam/policies/labs1.yaml` | Added `observabilityadmin:StartTelemetryEnrichment`, `cloudwatch:StartOTelEnrichment` permissions | diff --git a/manifests/modules/observability/otlp-metrics/otel/application-otel-metrics.yaml b/manifests/modules/observability/otlp-metrics/otel/application-otel-metrics.yaml new file mode 100644 index 0000000000..eabe601e45 --- /dev/null +++ b/manifests/modules/observability/otlp-metrics/otel/application-otel-metrics.yaml @@ -0,0 +1,73 @@ +apiVersion: cloudwatch.aws.amazon.com/v1alpha1 +kind: AmazonCloudWatchAgent +metadata: + name: application-metrics-collector + namespace: amazon-cloudwatch +spec: + mode: deployment + replicas: 1 + image: ${CWA_IMAGE} + serviceAccount: cloudwatch-agent + config: '{"agent":{"region":"${AWS_REGION}"}}' + otelConfig: | + exporters: + otlphttp/cw: + auth: + authenticator: sigv4auth/cw + endpoint: https://monitoring.${AWS_REGION}.amazonaws.com:443 + tls: + insecure: false + extensions: + sigv4auth/cw: + region: ${AWS_REGION} + service: monitoring + processors: + batch: + send_batch_max_size: 500 + send_batch_size: 500 + timeout: 10s + receivers: + prometheus: + config: + scrape_configs: + - job_name: orders + metrics_path: /actuator/prometheus + scrape_interval: 60s + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - orders + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "8080" + action: keep + - source_labels: [__meta_kubernetes_namespace] + target_label: Namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: PodName + - job_name: checkout + metrics_path: /metrics + scrape_interval: 60s + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - checkout + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "8080" + action: keep + - source_labels: [__meta_kubernetes_namespace] + target_label: Namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: PodName + service: + extensions: [sigv4auth/cw] + pipelines: + metrics: + exporters: [otlphttp/cw] + processors: [batch] + receivers: [prometheus] + + diff --git a/manifests/modules/observability/otlp-metrics/otel/kustomization.yaml b/manifests/modules/observability/otlp-metrics/otel/kustomization.yaml new file mode 100644 index 0000000000..dd28ece2b0 --- /dev/null +++ b/manifests/modules/observability/otlp-metrics/otel/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: amazon-cloudwatch +resources: + - application-otel-metrics.yaml + diff --git a/website/docs/observability/index.md b/website/docs/observability/index.md index 188d6e735d..22aaeb22f7 100644 --- a/website/docs/observability/index.md +++ b/website/docs/observability/index.md @@ -12,6 +12,7 @@ In this chapter, we'll cover how you can use AWS observability solutions integra - Control Plane and Pod Logs utilizing Fluentbit - Monitoring Metrics with CloudWatch Container Insights - Monitoring EKS Metrics with AMP and ADOT. +- Monitor Using Opentelemetry and PromQL Watch a video walk-through of the observability module with one of the module maintainers, Nirmal Mehta (AWS) here: diff --git a/website/docs/observability/open-source-metrics/opentemetry-promql.md b/website/docs/observability/open-source-metrics/opentemetry-promql.md new file mode 100644 index 0000000000..f47930943b --- /dev/null +++ b/website/docs/observability/open-source-metrics/opentemetry-promql.md @@ -0,0 +1,174 @@ +--- +title: "OpenTelemetry & PromQL" +sidebar_position: 60 +--- + +In this section we'll look at the following: + +- Enable OTel Enrichment: Enrich existing CloudWatch metrics with AWS resource attributes for unified PromQL querying +- Send Custom Metrics: Collect application metrics from petsite (.NET on EKS) and petfood (Rust on ECS) using the CloudWatch Agent +- Query with PromQL: Use Amazon CloudWatch Query Studio to query infrastructure, application, and enriched metrics +- Amazon Managed Grafana: Connect Amazon Managed Grafana and build a full-stack observability dashboard + +Let's Enable AWS vended metric enrichment. + +Each of the components in this workshop have been instrumented to provide Prometheus metrics using libraries relevant to the particular programming language or framework. We can look at an example of these metrics from the orders service like so: + +```bash +$ kubectl -n orders exec deployment/orders -- curl http://localhost:8080/actuator/prometheus +[...] +# HELP jdbc_connections_idle Number of established but idle connections. +# TYPE jdbc_connections_idle gauge +jdbc_connections_idle{name="reader",} 10.0 +jdbc_connections_idle{name="writer",} 10.0 +[...] +# HELP watch_orders_total The number of orders placed +# TYPE watch_orders_total counter +watch_orders_total{productId="510a0d7e-8e83-4193-b483-e27e09ddc34d",} 2.0 +watch_orders_total{productId="808a2de1-1aaa-4c25-a9b9-6612e8f29a38",} 1.0 +watch_orders_total{productId="*",} 3.0 +watch_orders_total{productId="6d62d909-f957-430e-8689-b5129c0bb75e",} 1.0 +``` + +The output from this command is verbose so the example above has been pruned to show: + +- System metric - How many JDBC connections are idle +- Application metric - How many orders have been placed through the retail store + +You can execute similar requests to other components, for example the checkout service: + +```bash +$ kubectl -n checkout exec deployment/checkout -- curl http://localhost:8080/metrics +[...] +# HELP nodejs_heap_size_total_bytes Process heap size from Node.js in bytes. +# TYPE nodejs_heap_size_total_bytes gauge +nodejs_heap_size_total_bytes 48668672 +[...] +``` + +In this lab we'll leverage ADOT to ingest the metrics for all the components and explore a dashboard to show the number of orders that have been placed. Let's take a look at the OpenTelemetry configuration used to scrape metrics from the application pods, specifically this section: + +```bash +$ kubectl -n other get opentelemetrycollector adot -o jsonpath='{.spec.config}' \ + | yq '.receivers.prometheus.config.scrape_configs[2]' +job_name: 'kubernetes-pods' +honor_labels: true +kubernetes_sd_configs: + - role: pod +relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow] + action: drop + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: replace + regex: (https?) + target_label: __scheme__ + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod + - source_labels: [__meta_kubernetes_pod_phase] + regex: Pending|Succeeded|Failed|Completed + action: drop +``` + +This configuration leverages the Prometheus [Kubernetes service discovery](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) mechanism to automatically discover all pods with specific annotations. This particular configuration will discover any pods with the annotation `prometheus.io/scrape`, and will enrich metrics it scrapes with Kubernetes metadata such as the namespace and pod name. + +We can check the annotations on the order component pods: + +```bash +$ kubectl get -o yaml -n orders deployment/orders | yq '.spec.template.metadata.annotations' +prometheus.io/path: /actuator/prometheus +prometheus.io/port: "8080" +prometheus.io/scrape: "true" +``` + +As we saw in the section regarding cluster metrics, these pod metrics will also be sent to AMP using the same OpenTelemetry exporter. + +Next use the below script to run a load generator which will place orders through the store and generate application metrics: + +```bash test=false +$ cat <.amazonaws.com` +4. Set **Service Provider** to `monitoring` +5. Set **Region** to your AWS Region +6. Choose **Default Credential Chain** as authentication provider +7. Click **Save & test** + +Refer to the official documentation: [CloudWatch PromQL](https://docs.aws.amazon.com/grafana/latest/userguide/cloudwatch-promql.html) diff --git a/website/docs/observability/opentelemetry-promql/enable-aws-metric.md b/website/docs/observability/opentelemetry-promql/enable-aws-metric.md new file mode 100644 index 0000000000..da73f17d91 --- /dev/null +++ b/website/docs/observability/opentelemetry-promql/enable-aws-metric.md @@ -0,0 +1,41 @@ +--- +title: "Enable AWS vended metric enrichment" +sidebar_position: 10 +--- + +To ingest and query OpenTelemetry metrics in Amazon CloudWatch, there are two account-level settings to enable, which can be done through the AWS console, AWS CLI, or AWS SDK. The first setting enables resource tag propagation from your AWS resources to your telemetry data, the same tags that are visible in AWS Resource Explorer. The second enables OpenTelemetry metrics ingestion for CloudWatch. + +Let's enable these two settings. You can use either the AWS Console or the AWS CLI. + + +**Architecture** + +Amazon CloudWatch exposes a regional OTLP endpoint that OpenTelemetry-compatible collectors and SDKs can send metrics to directly. Metrics are stored in a high-cardinality metrics store that retains OpenTelemetry metric types including counters, histograms, gauges, and up-down counters without conversion. When enrichment is enabled, CloudWatch automatically tags vended metrics with AWS resource context such as account ID, region, cluster ARN, and resource tags from AWS Resource Explorer. This enrichment requires no manual instrumentation, enabling you to query and filter across AWS accounts, regions, and resource tags using PromQL in Amazon CloudWatch Query Studio or Amazon Managed Grafana. + + + +**Option A: AWS Console** + +1. Open the Amazon CloudWatch console +2. In the left navigation, choose Settings +3. In the Resource tags on telemetry section, toggle Enable resource tags for telemetry to On +4. In the OTel metric ingestion section, toggle Enable OTel metric ingestion to On + +![Cloudwatch Settings](/docs/observability/opentelemetry-promql/enabled_otel_resource_tags.webp) + + +**Option B: AWS CLI** + +Enable resource tags for telemetry data + +```bash hook=cluster-logging +$ aws observabilityadmin start-telemetry-enrichment +``` + +Enable OpenTelemetry metrics ingestion for Amazon CloudWatch + + +```bash hook=cluster-logging +$ aws cloudwatch start-otel-enrichment +``` + diff --git a/website/docs/observability/opentelemetry-promql/index.md b/website/docs/observability/opentelemetry-promql/index.md new file mode 100644 index 0000000000..1b4f799967 --- /dev/null +++ b/website/docs/observability/opentelemetry-promql/index.md @@ -0,0 +1,61 @@ +--- +title: "OpenTelemetry & PromQL" +sidebar_position: 100 +sidebar_custom_props: { "module": true } +description: "Send OpenTelemetry metrics from any OpenTelemetry-instrumented application or collector directly to the Amazon CloudWatch OpenTelemetry" +--- + +::required-time + +:::tip Before you start +Prepare your environment for this section: + +```bash timeout=300 wait=30 +$ prepare-environment observability/otlp-metrics +``` + +::: + + +**OpenTelemetry metric ingestion** + +You can send OpenTelemetry metrics from any OpenTelemetry-instrumented application or collector directly to the Amazon CloudWatch OpenTelemetry endpoint `https://monitoring..amazonaws.com/v1/metrics` using open standards, through a single protocol. This means you can use standard OpenTelemetry SDKs and collectors to ship metrics to Amazon CloudWatch natively, with no custom exporters, no sidecars translating formats, and no intermediate services required. + +**Enriching existing CloudWatch vended metrics** + +Amazon CloudWatch allows OpenTelemetry metric enrichment of existing CloudWatch vended metrics from services like EC2, RDS, ECS, etc. With OpenTelemetry metric enrichment, you can query all your metrics, both OpenTelemetry metrics-ingested and AWS vended, using PromQL in a single place. + +**Container Insights with OpenTelemetry** + +The Amazon CloudWatch Observability EKS add-on collects Container Insights metrics via the OTLP pipeline, providing cluster, node, pod, and container-level metrics for Amazon EKS. These metrics flow through the same OTLP endpoint and are queryable with PromQL alongside your application and enriched vended metrics. + + + +**Architecture** + +Amazon CloudWatch natively supports ingestion of OpenTelemetry Protocol (OTLP) metrics and querying them with Prometheus Query Language (PromQL). This capability provides a high-cardinality metrics store that supports up to 150 labels per metric, enabling you to send rich, label-dense metrics directly to Amazon CloudWatch without conversion or truncation and without requiring any additional or intermediate infrastructure. Combined with automatic AWS vended metric enrichment, Amazon CloudWatch becomes a unified destination for infrastructure, container, and application metrics, all queryable with PromQL. + +![Architecture Diagram](/docs/observability/opentelemetry-promql/otel-architecture.webp) + +**Key Capabilities** + +| Capability | Description | +|------|--------| +| OpenTelemetry Ingestion | Send OpenTelemetry metrics to `https://monitoring..amazonaws.com/v1/metrics` | +| OTel Container Insights | Collect cluster, node, pod, and container metrics for Amazon EKS via the OTLP pipeline | +| PromQL Queries | Query OpenTelemetry metrics using PromQL in the Amazon CloudWatch Query Studio | +| SQL Queries | Query OpenTelemetry metrics using SQL syntax in the Amazon CloudWatch Query Studio | +| Enrichment | Enrich CloudWatch vended metrics with AWS resource attributes for unified PromQL querying | +| Grafana Integration | Integrate OpenTelemetry metrics as a data source in Grafana (Amazon Managed or open source)| +| Custom Metrics | Collect application-level Prometheus metrics from ECS services via the CloudWatch Agent OTLP pipeline | + + +**What this section covers** + +Here's what you'll learn: + +- **Enable OTel Enrichment:** Enrich existing CloudWatch metrics with AWS resource attributes for unified PromQL querying +- **Send Custom Metrics:** Collect application metrics from petsite (.NET on EKS) and petfood (Rust on ECS) using the CloudWatch Agent +- **Query with PromQL:** Use Amazon CloudWatch Query Studio to query infrastructure, application, and enriched metrics +- **Amazon Managed Grafana:** Connect Amazon Managed Grafana and build a full-stack observability dashboard + diff --git a/website/docs/observability/opentelemetry-promql/query-with-promql.md b/website/docs/observability/opentelemetry-promql/query-with-promql.md new file mode 100644 index 0000000000..b04676f338 --- /dev/null +++ b/website/docs/observability/opentelemetry-promql/query-with-promql.md @@ -0,0 +1,325 @@ +--- +title: "Query with PromQL in CloudWatch" +sidebar_position: 30 +--- + +Amazon CloudWatch Query Studio is a unified query interface that lets you explore metrics using both PromQL (Prometheus Query Language) and SQL syntax. This provides a unified interface to explore Container Insights infrastructure metrics and application metrics — all without needing a separate Prometheus server. It provides auto-completion, a metric browser, and a visual query builder so you can explore OpenTelemetry-ingested metrics, enriched AWS vended metrics, and custom application metrics from a single pane. + +In this section we'll explore how to query the metrics collected by the OpenTelemetry-based CloudWatch agent using PromQL directly in Amazon CloudWatch Query Studio. + + + +To get started, navigate to **Query Studio** in the CloudWatch console left navigation menu under the Metrics section, and select **PromQL** as the query language. + +## Verifying metrics collection + +First, let's confirm the CloudWatch agent is running and collecting metrics from the cluster: + +```bash +$ kubectl get pods -n amazon-cloudwatch +NAME READY STATUS RESTARTS AGE +amazon-cloudwatch-observability-controller-manager-7bf79bbjc8w5 1/1 Running 0 19m +application-metrics-collector-5bbb4675b8-6zsfb 1/1 Running 0 2m +cloudwatch-agent-cluster-scraper-7578f8fccb-8cxjm 1/1 Running 0 18m +cloudwatch-agent-pfsvk 1/1 Running 0 18m +cloudwatch-agent-wm6lp 1/1 Running 0 18m +cloudwatch-agent-zqvz8 1/1 Running 0 18m +kube-state-metrics-754d95978-7s5cp 1/1 Running 0 19m +node-exporter-78rgq 1/1 Running 0 19m +node-exporter-nlkc9 1/1 Running 0 19m +node-exporter-ztqcg 1/1 Running 0 19m +``` + +You should see `cloudwatch-agent` pods running on each node (as a DaemonSet), a cluster scraper, kube-state-metrics, node-exporter pods, and the `application-metrics-collector` deployment. These components work together to collect infrastructure and application metrics via OpenTelemetry and send them to CloudWatch. + +Metrics start appearing in Query Studio within 5 minutes of the pods being ready and you can query them using PromQL or SQL syntax based queries. Go ahead and try running these below sample queries. + +![CW Query Studio PromQL results](/docs/observability/opentelemetry-promql/cw-promql-query.webp) + +## Cluster-level metrics + +These queries provide a high-level view of overall cluster health and resource utilization. + +**Total cluster CPU usage** — aggregate CPU consumption across all containers: + +```text +sum(rate(container_cpu_usage_seconds_total[5m])) +``` + +**Average CPU usage per namespace** — shows the mean CPU consumption per namespace, useful for baselining: + +```text +avg by(k8s_namespace_name) (rate(container_cpu_usage_seconds_total[5m])) +``` + +**Maximum memory usage by namespace** — identifies the peak memory consumer in each namespace: + +```text +max by(k8s_namespace_name) (container_memory_working_set_bytes) +``` + +**CPU usage by namespace** — total CPU broken down by namespace: + +```text +sum by(k8s_namespace_name) (rate(container_cpu_usage_seconds_total[5m])) +``` + +## Node metrics + +These queries focus on individual node health and capacity, useful for identifying hotspots or underutilized nodes. + +**Node CPU utilization percentage** — shows how much of each node's CPU capacity is being used: + +```text +sum by(kubernetes_io_hostname) (rate(node_cpu_seconds_total{mode!="idle"}[5m])) / sum by(kubernetes_io_hostname) (rate(node_cpu_seconds_total[5m])) * 100 +``` + +**Minimum available memory per node** — find the node closest to running out of memory: + +```text +min by(kubernetes_io_hostname) (node_memory_MemAvailable_bytes) +``` + +**Node filesystem usage percentage** — disk utilization as a percentage per node: + +```text +(sum by(kubernetes_io_hostname) (node_filesystem_size_bytes - node_filesystem_avail_bytes)) / sum by(kubernetes_io_hostname) (node_filesystem_size_bytes) * 100 +``` + +**Node count by instance type** — useful for understanding your cluster's compute composition: + +```text +count by(node_kubernetes_io_instance_type) (kube_node_info) +``` + +## Pod metrics + +Pod-level queries help you understand workload distribution and scheduling patterns. + +**Pod count by namespace** — gives a quick view of workload density per namespace: + +```text +sum by(k8s_namespace_name) (kube_pod_info) +``` + +**Pod count by availability zone** — shows how your pods are distributed across AZs: + +```text +count by(topology_kubernetes_io_zone) (kube_pod_info) +``` + +**Pod count by node** — identifies if pods are evenly spread or concentrated on specific nodes: + +```text +count by(kubernetes_io_hostname) (kube_pod_info) +``` + +**Running pods per namespace** — useful for verifying expected replica counts: + +```text +count by(k8s_namespace_name) (container_cpu_usage_seconds_total) +``` + +## Container metrics + +Container-level metrics give you insight in to individual workload resource consumption. + +**Top 5 pods by CPU usage** — uses topk to surface the heaviest consumers: + +```text +topk(5, sum by(k8s_namespace_name, k8s_pod_name) (rate(container_cpu_usage_seconds_total[5m]))) +``` + +**Bottom 5 pods by memory** — find the lightest workloads using bottomk: + +```text +bottomk(5, sum by(k8s_namespace_name, k8s_pod_name) (container_memory_working_set_bytes)) +``` + +**CPU usage rate per pod** — drill down to individual pod CPU consumption: + +```text +sum by(k8s_pod_name) (rate(container_cpu_usage_seconds_total[5m])) +``` + +**Average memory per pod** — useful for right-sizing resource requests: + +```text +avg by(k8s_pod_name) (container_memory_working_set_bytes) +``` + +## Network metrics + +Network metrics help identify traffic patterns, bottlenecks, and imbalanced load across nodes. + +**Network receive rate by node** — identifies which nodes are handling the most inbound traffic: + +```text +sum by(kubernetes_io_hostname) (rate(node_network_receive_bytes_total[5m])) +``` + +**Network transmit rate by node** — outbound traffic per node: + +```text +sum by(kubernetes_io_hostname) (rate(node_network_transmit_bytes_total[5m])) +``` + +**Total cluster network throughput** — combined inbound and outbound across all nodes: + +```text +sum(rate(node_network_receive_bytes_total[5m])) + sum(rate(node_network_transmit_bytes_total[5m])) +``` + +**Maximum network error rate across nodes** — surfaces the node with the worst network health: + +```text +max by(kubernetes_io_hostname) (rate(node_network_receive_errs_total[5m]) + rate(node_network_transmit_errs_total[5m])) +``` + +## Application and service metrics + +To query application-level metrics, we first need to deploy a dedicated CloudWatch Agent collector that scrapes Prometheus endpoints exposed by the workshop services. This collector is configured to scrape the `orders` service (Java/Spring Boot) and the `checkout` service (Node.js). + +Deploy the application metrics collector: + +```bash +$ export CWA_IMAGE=$(kubectl get daemonset -n amazon-cloudwatch cloudwatch-agent -o jsonpath='{.spec.template.spec.containers[0].image}') +$ kubectl kustomize ~/environment/eks-workshop/modules/observability/otlp-metrics/otel \ + | envsubst | kubectl apply -f - +amazoncloudwatchagent.cloudwatch.aws.amazon.com/application-metrics-collector created +``` + +Next, deploy a load generator to produce application traffic and generate order metrics: + +```bash test=false +$ cat <