Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion components/egress/docs/opentelemetry.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,45 @@ Metric export is enabled only when at least one OTLP endpoint is set.

If both are unset, egress keeps metrics local (no OTLP export).

### Automatic Egress Allow Rule

When an OTLP destination is configured — the endpoint env vars below, or the
exporter fallback node IP (`HOST_IP` / `/etc/hostinfo`) when both are unset —
egress automatically injects an always-allow egress rule for that host
(domain or IP, any port), so telemetry export works under the default deny-all
policy without manually managing allowlist rules. This also covers the egress
sidecar's own metric export, which shares the sandbox network namespace and
would otherwise be blocked by its own egress chain.

- The rule follows the standard precedence: `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`
wins over `OTEL_EXPORTER_OTLP_ENDPOINT`; the fallback node IP applies when
neither is set.
- The host is taken from the endpoint URL (`https://host:4318/v1/metrics`),
`host:port`, or bare `host` forms; a trailing root dot on FQDNs is trimmed to
match DNS policy normalization.
- The rule lives in the always-allow layer: it survives user `POST`/`PATCH`/`DELETE`
policy updates and always-rule file reloads. Operators can still block the target
with `deny.always`, which takes precedence.
- Rules are host-scoped (any port), matching the egress rule model; ports are not
enforced per rule.

> **Note**: use a fully-qualified service name or an IP in the endpoint.
> Single-label names (e.g. `otel-collector`) are subject to resolver
> search-domain expansion, and the deny-all DNS proxy answers the expanded
> names (e.g. `otel-collector.<ns>.svc.cluster.local`) with NXDOMAIN without
> falling back to the bare name, so the auto-generated exact-host allow rule
> would not be reached.

### Minimal Example

```bash
export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://otel-collector:4318"
export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://otel-collector.sandbox.svc.cluster.local:4318"
```

An IP endpoint works as well:

```bash
export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://10.0.0.5:4318"
```

### Service Name
Expand Down
1 change: 1 addition & 0 deletions components/egress/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ func main() {
if err != nil {
log.Fatalf("failed to load always allow/deny rule files: %v", err)
}
alwaysAllow = withTelemetryAllow(alwaysAllow)

allowIPs := allowIps()
mode := parseMode()
Expand Down
2 changes: 2 additions & 0 deletions components/egress/policy_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,8 @@ func (s *policyServer) reloadAlwaysRules() (bool, error) {
if !changed {
return false, nil
}
allow = withTelemetryAllow(allow)
s.setAlwaysRules(deny, allow)
s.proxy.UpdateAlwaysRules(deny, allow)
return true, nil
}
Expand Down
60 changes: 60 additions & 0 deletions components/egress/telemetry_allow.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright 2026 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"github.com/alibaba/opensandbox/egress/pkg/log"
"github.com/alibaba/opensandbox/egress/pkg/policy"
inttelemetry "github.com/alibaba/opensandbox/internal/telemetry"
)

// telemetryAllowRules returns an always-allow egress rule for the OTLP
// destination the exporter will dial, so metric export works under the default
// deny-all policy without operator-provided allowlist rules. The destination
// is the configured endpoint
// (OTEL_EXPORTER_OTLP_METRICS_ENDPOINT / OTEL_EXPORTER_OTLP_ENDPOINT) or, when
// neither is set, the exporter fallback node IP (HOST_IP / /etc/hostinfo). The
// rule targets the host (any port), matching the egress rule model. Operators
// can still block the target via deny.always, which takes precedence. Returns
// nil when no OTLP destination is configured.
func telemetryAllowRules() []policy.EgressRule {
host, port, ok := inttelemetry.OTLPEndpointHostPort()
if !ok {
host, port, ok = inttelemetry.OTLPEndpointFallbackHostPort()
Comment thread
Pangjiping marked this conversation as resolved.
}
if !ok {
return nil
}
rule, err := policy.ParseValidatedEgressRule(policy.ActionAllow, host)
Comment thread
Pangjiping marked this conversation as resolved.
if err != nil {
log.Warnf("telemetry: skipping auto egress allow for OTLP endpoint host %q: %v", host, err)
return nil
}
log.Infof("telemetry: auto-allowing egress to OTLP endpoint %s:%s (deny.always can override)", host, port)
return []policy.EgressRule{rule}
}

// withTelemetryAllow appends the auto-generated OTLP allow rule(s) to the
// always-allow list so every effective-policy merge (startup, policy updates,
// always-file reloads) keeps telemetry egress open.
func withTelemetryAllow(allow []policy.EgressRule) []policy.EgressRule {
rules := telemetryAllowRules()
if len(rules) == 0 {
return allow
}
out := make([]policy.EgressRule, 0, len(allow)+len(rules))
out = append(out, allow...)
return append(out, rules...)
}
116 changes: 116 additions & 0 deletions components/egress/telemetry_allow_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
// Copyright 2026 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"testing"

"github.com/alibaba/opensandbox/egress/pkg/policy"
"github.com/stretchr/testify/require"
)

func TestTelemetryAllowRulesUnconfigured(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
require.Nil(t, telemetryAllowRules())

existing := []policy.EgressRule{{Action: policy.ActionAllow, Target: "a.example.com"}}
require.Equal(t, existing, withTelemetryAllow(existing), "no telemetry rules must not mutate the input")
}

func TestTelemetryAllowRulesFromMetricsEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "https://collector.example:4318/v1/metrics")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, policy.ActionAllow, rules[0].Action)
require.Equal(t, "collector.example", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
require.Equal(t, policy.ActionAllow, merged.Evaluate("collector.example."), "domain rule must allow DNS resolution")
allowV4, allowV6, _, _ := merged.StaticIPSets()
require.Empty(t, allowV4)
require.Empty(t, allowV6)
}

func TestTelemetryAllowRulesFallbackEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "otel-collector:4318")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "otel-collector", rules[0].Target)
}

func TestTelemetryAllowRulesFallbackNodeIP(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
t.Setenv("HOST_IP", "10.0.0.9")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "10.0.0.9", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
allowV4, allowV6, _, _ := merged.StaticIPSets()
require.Equal(t, []string{"10.0.0.9"}, allowV4, "fallback node IP must land in the static allow v4 set")
require.Empty(t, allowV6)
}

func TestTelemetryAllowRulesFQDNTrailingDot(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://otel-collector.ns.svc.cluster.local.:4318")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "otel-collector.ns.svc.cluster.local", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
require.Equal(t, policy.ActionAllow, merged.Evaluate("otel-collector.ns.svc.cluster.local."), "trailing-dot host must match DNS policy normalization")
require.Equal(t, policy.ActionDeny, merged.Evaluate("other.ns.svc.cluster.local."))
}

func TestTelemetryAllowRulesIPEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://10.0.0.5:4317")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "10.0.0.5", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
allowV4, allowV6, _, _ := merged.StaticIPSets()
require.Equal(t, []string{"10.0.0.5"}, allowV4, "IP target must land in the static allow v4 set")
require.Empty(t, allowV6)
}

func TestTelemetryAllowRulesInvalidEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
require.Nil(t, telemetryAllowRules())
}

func TestWithTelemetryAllowAppends(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "collector.example:4318")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
existingRule, err := policy.ParseValidatedEgressRule(policy.ActionAllow, "a.example.com")
require.NoError(t, err)
existing := []policy.EgressRule{existingRule}
rules := withTelemetryAllow(existing)
require.Len(t, rules, 2)
require.Equal(t, "a.example.com", rules[0].Target)
require.Equal(t, "collector.example", rules[1].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
require.Equal(t, policy.ActionDeny, merged.Evaluate("other.example.com."))
require.Equal(t, policy.ActionAllow, merged.Evaluate("a.example.com."))
require.Equal(t, policy.ActionAllow, merged.Evaluate("collector.example."))
}
72 changes: 47 additions & 25 deletions components/execd/pkg/web/controller/pty_ws.go
Original file line number Diff line number Diff line change
Expand Up @@ -500,38 +500,60 @@ func ptyViewerClientReadLoop(

switch msgType {
case websocket.BinaryMessage:
if len(data) > 0 && data[0] == model.BinStdin {
if !readOnlyError() {
return
}
if !ptyViewerHandleBinaryMessage(data, readOnlyError) {
return
}
case websocket.TextMessage:
var frame model.ClientFrame
if json.Unmarshal(data, &frame) != nil {
continue
}
switch frame.Type {
case "stdin", "signal", "resize":
if !readOnlyError() {
return
}
case "ping":
if err := writeJSON(model.ServerFrame{Type: "pong"}); err != nil {
cancelOnce()
}
default:
if err := writeJSON(model.ServerFrame{
Type: "error",
Code: model.WSErrCodeInvalidFrame,
Error: fmt.Sprintf("unknown frame type %q", frame.Type),
}); err != nil {
cancelOnce()
}
if !ptyViewerHandleTextMessage(data, writeJSON, readOnlyError, cancelOnce) {
return
}
}
}
}

// ptyViewerHandleBinaryMessage reports stdin payloads on a read-only viewer;
// returns false when the read loop should exit.
func ptyViewerHandleBinaryMessage(data []byte, readOnlyError func() bool) bool {
if len(data) > 0 && data[0] == model.BinStdin {
return readOnlyError()
}
return true
}

// ptyViewerHandleTextMessage handles client frames on a read-only viewer;
// returns false when the read loop should exit.
func ptyViewerHandleTextMessage(data []byte, writeJSON func(any) error, readOnlyError func() bool, cancelOnce func()) bool {
var frame model.ClientFrame
if json.Unmarshal(data, &frame) != nil {
return true
}
switch frame.Type {
case "stdin", "signal", "resize":
return readOnlyError()
case "ping":
ptyViewerReplyPong(writeJSON, cancelOnce)
default:
ptyViewerReplyInvalidFrame(writeJSON, cancelOnce, frame.Type)
}
return true
}

func ptyViewerReplyPong(writeJSON func(any) error, cancelOnce func()) {
if err := writeJSON(model.ServerFrame{Type: "pong"}); err != nil {
cancelOnce()
}
}

func ptyViewerReplyInvalidFrame(writeJSON func(any) error, cancelOnce func(), frameType string) {
if err := writeJSON(model.ServerFrame{
Type: "error",
Code: model.WSErrCodeInvalidFrame,
Error: fmt.Sprintf("unknown frame type %q", frameType),
}); err != nil {
cancelOnce()
}
}

// ptyPingLoop sends periodic WebSocket pings until cancelCh is closed.
func ptyPingLoop(conn *websocket.Conn, connMu *sync.Mutex, cancelCh <-chan struct{}, cancelOnce func()) {
t := time.NewTicker(wsPingInterval)
Expand Down
Loading
Loading