Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion components/egress/docs/opentelemetry.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,47 @@ Metric export is enabled only when at least one OTLP endpoint is set.

If both are unset, egress keeps metrics local (no OTLP export).

### Automatic Egress Allow Rule

When an OTLP destination is configured — the endpoint env vars below, or the
exporter fallback node IP (`HOST_IP` / `/etc/hostinfo`) when both are unset —
egress automatically injects an always-allow egress rule for that host
(domain or IP, any port), so telemetry export works under the default deny-all
policy without manually managing allowlist rules. This also covers the egress
sidecar's own metric export, which shares the sandbox network namespace and
would otherwise be blocked by its own egress chain.

- The rule follows the standard precedence: `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`
wins over `OTEL_EXPORTER_OTLP_ENDPOINT`; the fallback node IP applies only
when neither is set. A set-but-invalid endpoint never falls back (the
exporter does not either), so no rule is injected in that case.
- The endpoint must be a URL (`https://host:4318/v1/metrics`) — the
`otlpmetrichttp` env-var form. Bare `host:port` or `host` values are not
accepted (the exporter parses them as opaque URLs with an empty host); a
trailing root dot on FQDNs is trimmed to match DNS policy normalization.
- The rule lives in the always-allow layer: it survives user `POST`/`PATCH`/`DELETE`
policy updates and always-rule file reloads. Operators can still block the target
with `deny.always`, which takes precedence.
- Rules are host-scoped (any port), matching the egress rule model; ports are not
enforced per rule.

> **Note**: use a fully-qualified service name or an IP in the endpoint.
> Single-label names (e.g. `otel-collector`) are subject to resolver
> search-domain expansion, and the deny-all DNS proxy answers the expanded
> names (e.g. `otel-collector.<ns>.svc.cluster.local`) with NXDOMAIN without
> falling back to the bare name, so the auto-generated exact-host allow rule
> would not be reached.

### Minimal Example

```bash
export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://otel-collector:4318"
export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://otel-collector.sandbox.svc.cluster.local:4318"
```

An IP endpoint works as well:

```bash
export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://10.0.0.5:4318"
```

### Service Name
Expand Down
1 change: 1 addition & 0 deletions components/egress/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ func main() {
if err != nil {
log.Fatalf("failed to load always allow/deny rule files: %v", err)
}
alwaysAllow = withTelemetryAllow(alwaysAllow)

allowIPs := allowIps()
mode := parseMode()
Expand Down
2 changes: 2 additions & 0 deletions components/egress/policy_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,8 @@ func (s *policyServer) reloadAlwaysRules() (bool, error) {
if !changed {
return false, nil
}
allow = withTelemetryAllow(allow)
s.setAlwaysRules(deny, allow)
s.proxy.UpdateAlwaysRules(deny, allow)
return true, nil
}
Expand Down
67 changes: 67 additions & 0 deletions components/egress/telemetry_allow.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copyright 2026 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"github.com/alibaba/opensandbox/egress/pkg/log"
"github.com/alibaba/opensandbox/egress/pkg/policy"
inttelemetry "github.com/alibaba/opensandbox/internal/telemetry"
)

// telemetryAllowRules returns an always-allow egress rule for the OTLP
// destination the exporter will dial, so metric export works under the default
// deny-all policy without operator-provided allowlist rules. The destination
// is the endpoint env var
// (OTEL_EXPORTER_OTLP_METRICS_ENDPOINT / OTEL_EXPORTER_OTLP_ENDPOINT, URL form
// as required by otlpmetrichttp) or, only when neither is set, the exporter
// fallback node IP (HOST_IP / /etc/hostinfo). A set-but-unparseable endpoint
// is not treated as unset: the exporter never falls back to the node IP in
// that case, so no rule is injected. The rule targets the host (any port),
// matching the egress rule model. Operators can still block the target via
// deny.always, which takes precedence. Returns nil when no OTLP destination
// is configured.
func telemetryAllowRules() []policy.EgressRule {
host, port, ok := inttelemetry.OTLPEndpointHostPort()
if !ok {
if inttelemetry.OTLPEndpointEnvSet() {
log.Warnf("telemetry: configured OTLP endpoint is not a valid URL; skipping auto egress allow")
return nil
}
host, port, ok = inttelemetry.OTLPEndpointFallbackHostPort()
Comment thread
Pangjiping marked this conversation as resolved.
}
if !ok {
return nil
}
rule, err := policy.ParseValidatedEgressRule(policy.ActionAllow, host)
Comment thread
Pangjiping marked this conversation as resolved.
if err != nil {
log.Warnf("telemetry: skipping auto egress allow for OTLP endpoint host %q: %v", host, err)
return nil
}
log.Infof("telemetry: auto-allowing egress to OTLP endpoint %s:%s (deny.always can override)", host, port)
return []policy.EgressRule{rule}
}

// withTelemetryAllow appends the auto-generated OTLP allow rule(s) to the
// always-allow list so every effective-policy merge (startup, policy updates,
// always-file reloads) keeps telemetry egress open.
func withTelemetryAllow(allow []policy.EgressRule) []policy.EgressRule {
rules := telemetryAllowRules()
if len(rules) == 0 {
return allow
}
out := make([]policy.EgressRule, 0, len(allow)+len(rules))
out = append(out, allow...)
return append(out, rules...)
}
128 changes: 128 additions & 0 deletions components/egress/telemetry_allow_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright 2026 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"testing"

"github.com/alibaba/opensandbox/egress/pkg/policy"
"github.com/stretchr/testify/require"
)

func TestTelemetryAllowRulesUnconfigured(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
require.Nil(t, telemetryAllowRules())

existing := []policy.EgressRule{{Action: policy.ActionAllow, Target: "a.example.com"}}
require.Equal(t, existing, withTelemetryAllow(existing), "no telemetry rules must not mutate the input")
}

func TestTelemetryAllowRulesFromMetricsEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "https://collector.example:4318/v1/metrics")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, policy.ActionAllow, rules[0].Action)
require.Equal(t, "collector.example", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
require.Equal(t, policy.ActionAllow, merged.Evaluate("collector.example."), "domain rule must allow DNS resolution")
allowV4, allowV6, _, _ := merged.StaticIPSets()
require.Empty(t, allowV4)
require.Empty(t, allowV6)
}

func TestTelemetryAllowRulesFallbackEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel-collector:4318")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "otel-collector", rules[0].Target)
}

func TestTelemetryAllowRulesFallbackNodeIP(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
t.Setenv("HOST_IP", "10.0.0.9")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "10.0.0.9", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
allowV4, allowV6, _, _ := merged.StaticIPSets()
require.Equal(t, []string{"10.0.0.9"}, allowV4, "fallback node IP must land in the static allow v4 set")
require.Empty(t, allowV6)
}

func TestTelemetryAllowRulesFQDNTrailingDot(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://otel-collector.ns.svc.cluster.local.:4318")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "otel-collector.ns.svc.cluster.local", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
require.Equal(t, policy.ActionAllow, merged.Evaluate("otel-collector.ns.svc.cluster.local."), "trailing-dot host must match DNS policy normalization")
require.Equal(t, policy.ActionDeny, merged.Evaluate("other.ns.svc.cluster.local."))
}

func TestTelemetryAllowRulesIPEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://10.0.0.5:4317")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
rules := telemetryAllowRules()
require.Len(t, rules, 1)
require.Equal(t, "10.0.0.5", rules[0].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
allowV4, allowV6, _, _ := merged.StaticIPSets()
require.Equal(t, []string{"10.0.0.5"}, allowV4, "IP target must land in the static allow v4 set")
require.Empty(t, allowV6)
}

func TestTelemetryAllowRulesInvalidEndpoint(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
require.Nil(t, telemetryAllowRules())
}

func TestTelemetryAllowRulesInvalidEndpointSkipsNodeIPFallback(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
t.Setenv("HOST_IP", "10.0.0.9")
require.Nil(t, telemetryAllowRules(), "configured-but-invalid endpoint must not open node-IP egress")

t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "")
rules := telemetryAllowRules()
require.Len(t, rules, 1, "unset endpoint should fall back to the node IP")
require.Equal(t, "10.0.0.9", rules[0].Target)
}

func TestWithTelemetryAllowAppends(t *testing.T) {
t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "https://collector.example:4318")
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
existingRule, err := policy.ParseValidatedEgressRule(policy.ActionAllow, "a.example.com")
require.NoError(t, err)
existing := []policy.EgressRule{existingRule}
rules := withTelemetryAllow(existing)
require.Len(t, rules, 2)
require.Equal(t, "a.example.com", rules[0].Target)
require.Equal(t, "collector.example", rules[1].Target)

merged := policy.MergeAlwaysOverlay(policy.DefaultDenyPolicy(), nil, rules)
require.Equal(t, policy.ActionDeny, merged.Evaluate("other.example.com."))
require.Equal(t, policy.ActionAllow, merged.Evaluate("a.example.com."))
require.Equal(t, policy.ActionAllow, merged.Evaluate("collector.example."))
}
72 changes: 47 additions & 25 deletions components/execd/pkg/web/controller/pty_ws.go
Original file line number Diff line number Diff line change
Expand Up @@ -500,38 +500,60 @@ func ptyViewerClientReadLoop(

switch msgType {
case websocket.BinaryMessage:
if len(data) > 0 && data[0] == model.BinStdin {
if !readOnlyError() {
return
}
if !ptyViewerHandleBinaryMessage(data, readOnlyError) {
return
}
case websocket.TextMessage:
var frame model.ClientFrame
if json.Unmarshal(data, &frame) != nil {
continue
}
switch frame.Type {
case "stdin", "signal", "resize":
if !readOnlyError() {
return
}
case "ping":
if err := writeJSON(model.ServerFrame{Type: "pong"}); err != nil {
cancelOnce()
}
default:
if err := writeJSON(model.ServerFrame{
Type: "error",
Code: model.WSErrCodeInvalidFrame,
Error: fmt.Sprintf("unknown frame type %q", frame.Type),
}); err != nil {
cancelOnce()
}
if !ptyViewerHandleTextMessage(data, writeJSON, readOnlyError, cancelOnce) {
return
}
}
}
}

// ptyViewerHandleBinaryMessage reports stdin payloads on a read-only viewer;
// returns false when the read loop should exit.
func ptyViewerHandleBinaryMessage(data []byte, readOnlyError func() bool) bool {
if len(data) > 0 && data[0] == model.BinStdin {
return readOnlyError()
}
return true
}

// ptyViewerHandleTextMessage handles client frames on a read-only viewer;
// returns false when the read loop should exit.
func ptyViewerHandleTextMessage(data []byte, writeJSON func(any) error, readOnlyError func() bool, cancelOnce func()) bool {
var frame model.ClientFrame
if json.Unmarshal(data, &frame) != nil {
return true
}
switch frame.Type {
case "stdin", "signal", "resize":
return readOnlyError()
case "ping":
ptyViewerReplyPong(writeJSON, cancelOnce)
default:
ptyViewerReplyInvalidFrame(writeJSON, cancelOnce, frame.Type)
}
return true
}

func ptyViewerReplyPong(writeJSON func(any) error, cancelOnce func()) {
if err := writeJSON(model.ServerFrame{Type: "pong"}); err != nil {
cancelOnce()
}
}

func ptyViewerReplyInvalidFrame(writeJSON func(any) error, cancelOnce func(), frameType string) {
if err := writeJSON(model.ServerFrame{
Type: "error",
Code: model.WSErrCodeInvalidFrame,
Error: fmt.Sprintf("unknown frame type %q", frameType),
}); err != nil {
cancelOnce()
}
}

// ptyPingLoop sends periodic WebSocket pings until cancelCh is closed.
func ptyPingLoop(conn *websocket.Conn, connMu *sync.Mutex, cancelCh <-chan struct{}, cancelOnce func()) {
t := time.NewTicker(wsPingInterval)
Expand Down
Loading
Loading