Skip to content

fix(localdns): handle exporter client disconnects #8791

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jingwenw15 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(localdns): handle exporter client disconnects #8791

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 43 additions & 23 deletions parts/linux/cloud-init/artifacts/localdns_exporter.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,32 @@ set -euo pipefail
# This avoids the exporter needing to query systemd over D-Bus, which fails under DynamicUser=yes
# ("Transport endpoint is not connected") due to namespace isolation.

# A scrape client may close the socket after reading enough data (for example, a probe
# that only checks status/headers). Treat that as a normal connection termination so
# socket-activated worker units do not remain in a failed state.
trap 'exit 0' PIPE

# Pre-generated .prom files written by localdns.sh
# LOCALDNS_SCRIPT_PATH is set via Environment= in localdns-exporter@.service
LOCALDNS_SCRIPT_PATH="${LOCALDNS_SCRIPT_PATH:-/opt/azure/containers/localdns}"
RESOURCES_PROM_FILE="${LOCALDNS_SCRIPT_PATH}/resources.prom"
FORWARD_IPS_PROM_FILE="${LOCALDNS_SCRIPT_PATH}/forward_ips.prom"

emit() {
printf "%s\n" "$*" 2>/dev/null || exit 0
}
Comment on lines +27 to +29

emit_raw() {
printf "%b" "$1" 2>/dev/null || exit 0
}

emit_file() {
local file="$1"
while IFS= read -r line || [ -n "$line" ]; do
emit "$line"
done < "$file"
}

# Read the HTTP request line to extract the path
# Format: "GET /metrics HTTP/1.1"
# Handle read failure gracefully (client disconnected, incomplete request, or timeout)
Expand All @@ -30,43 +50,43 @@ REQUEST_PATH=$(echo "$REQUEST_LINE" | awk '{print $2}')

# Only serve metrics at /metrics endpoint (Prometheus convention)
if [ "$REQUEST_PATH" != "/metrics" ]; then
printf "HTTP/1.1 404 Not Found\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n"
echo "404 Not Found - Metrics available at /metrics"
emit_raw "HTTP/1.1 404 Not Found\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n"
emit "404 Not Found - Metrics available at /metrics"
exit 0
fi

# Output HTTP Response in Prometheus Exposition Format
# Note: The empty line after headers is required by HTTP protocol
printf "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nConnection: close\r\n\r\n"
emit_raw "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nConnection: close\r\n\r\n"

# Resource metrics (service status, CPU, memory) — pre-generated by localdns.sh
if [ -f "$RESOURCES_PROM_FILE" ]; then
cat "$RESOURCES_PROM_FILE"
emit_file "$RESOURCES_PROM_FILE"
else
# Fallback if .prom file doesn't exist yet (before first watchdog tick)
echo "# HELP localdns_service_status CoreDNS process status (1=active, 0=inactive)"
echo "# TYPE localdns_service_status gauge"
echo "localdns_service_status{status=\"unknown\"} 0"
echo "# HELP localdns_memory_usage_bytes Current memory usage in bytes"
echo "# TYPE localdns_memory_usage_bytes gauge"
echo "localdns_memory_usage_bytes 0"
echo "# HELP localdns_cpu_usage_seconds_total Total CPU time consumed in Seconds"
echo "# TYPE localdns_cpu_usage_seconds_total counter"
echo "localdns_cpu_usage_seconds_total 0.000000000"
echo "# HELP localdns_metrics_last_update_timestamp_seconds Unix timestamp of last metrics generation"
echo "# TYPE localdns_metrics_last_update_timestamp_seconds gauge"
echo "localdns_metrics_last_update_timestamp_seconds 0"
emit "# HELP localdns_service_status CoreDNS process status (1=active, 0=inactive)"
emit "# TYPE localdns_service_status gauge"
emit "localdns_service_status{status=\"unknown\"} 0"
emit "# HELP localdns_memory_usage_bytes Current memory usage in bytes"
emit "# TYPE localdns_memory_usage_bytes gauge"
emit "localdns_memory_usage_bytes 0"
emit "# HELP localdns_cpu_usage_seconds_total Total CPU time consumed in Seconds"
emit "# TYPE localdns_cpu_usage_seconds_total counter"
emit "localdns_cpu_usage_seconds_total 0.000000000"
emit "# HELP localdns_metrics_last_update_timestamp_seconds Unix timestamp of last metrics generation"
emit "# TYPE localdns_metrics_last_update_timestamp_seconds gauge"
emit "localdns_metrics_last_update_timestamp_seconds 0"
fi

# Forward IP info metrics (VnetDNS, KubeDNS) — pre-generated by localdns.sh
if [ -f "$FORWARD_IPS_PROM_FILE" ]; then
cat "$FORWARD_IPS_PROM_FILE"
emit_file "$FORWARD_IPS_PROM_FILE"
else
# Fallback if .prom file doesn't exist yet
echo "# HELP localdns_vnetdns_forward_info VnetDNS forward plugin IP address from corefile"
echo "# TYPE localdns_vnetdns_forward_info gauge"
echo "localdns_vnetdns_forward_info{ip=\"unknown\",block=\"none\",status=\"file_missing\"} 0"
echo "# HELP localdns_kubedns_forward_info KubeDNS forward plugin IP address from corefile"
echo "# TYPE localdns_kubedns_forward_info gauge"
echo "localdns_kubedns_forward_info{ip=\"unknown\",block=\"none\",status=\"file_missing\"} 0"
emit "# HELP localdns_vnetdns_forward_info VnetDNS forward plugin IP address from corefile"
emit "# TYPE localdns_vnetdns_forward_info gauge"
emit "localdns_vnetdns_forward_info{ip=\"unknown\",block=\"none\",status=\"file_missing\"} 0"
emit "# HELP localdns_kubedns_forward_info KubeDNS forward plugin IP address from corefile"
emit "# TYPE localdns_kubedns_forward_info gauge"
emit "localdns_kubedns_forward_info{ip=\"unknown\",block=\"none\",status=\"file_missing\"} 0"
fi
24 changes: 24 additions & 0 deletions spec/parts/linux/cloud-init/artifacts/localdns_exporter_spec.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,30 @@ Describe 'localdns_exporter.sh HTTP request routing'
The output should equal ""
End

It 'should exit cleanly when client closes during metrics response'
When run bash -o pipefail -c '
tmp_dir=$(mktemp -d)
trap "rm -rf ${tmp_dir}" EXIT
{
echo "# HELP localdns_service_status CoreDNS process status (1=active, 0=inactive)"
echo "# TYPE localdns_service_status gauge"
for i in $(seq 1 1000); do
echo "localdns_service_status{status=\"running\",sample=\"${i}\"} 1"
done
} > "${tmp_dir}/resources.prom"
{
echo "# HELP localdns_vnetdns_forward_info VnetDNS forward plugin IP address from corefile"
echo "# TYPE localdns_vnetdns_forward_info gauge"
echo "localdns_vnetdns_forward_info{ip=\"168.63.129.16\",block=\".:53\",status=\"ok\"} 1"
echo "# HELP localdns_kubedns_forward_info KubeDNS forward plugin IP address from corefile"
echo "# TYPE localdns_kubedns_forward_info gauge"
echo "localdns_kubedns_forward_info{ip=\"10.0.0.10\",block=\"cluster.local:53\",status=\"ok\"} 1"
} > "${tmp_dir}/forward_ips.prom"
printf "GET /metrics HTTP/1.1\r\n\r\n" | LOCALDNS_SCRIPT_PATH="${tmp_dir}" '"$SCRIPT_PATH"' | head -n 1 >/dev/null
'
The status should be success
End

It 'should return 200 and Prometheus metrics for /metrics path'
When run bash -c "echo 'GET /metrics HTTP/1.1' | $SCRIPT_PATH"
The status should be success
Expand Down
Loading