From f91f2948de525c5ea9a40ce86648613936a838a3 Mon Sep 17 00:00:00 2001 From: agentydragon-agent <260791330+agentydragon-agent@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:03:01 -0800 Subject: [PATCH 1/5] Route Runtime callers through the stable Gateway Service --- agentplane/app/README.md | 2 +- agentplane/app/internal/bridge/bridge.go | 4 +- agentplane/app/internal/bridge/runner.go | 6 +- .../app/internal/settings/config_test.go | 4 +- agentplane/app/settings.py | 5 +- agentplane/central_sampler/README.md | 8 +- agentplane/central_sampler/SPEC.md | 6 +- agentplane/central_sampler/cmd/main_test.go | 2 +- .../internal/settings/settings_test.go | 4 +- agentplane/docs/runner_control.md | 13 +-- agentplane/docs/system_overview.md | 3 +- agentplane/notification_service/README.md | 4 +- agentplane/notification_service/config.go | 4 +- .../notification_service/config_test.go | 8 +- .../internal/rpc/runner.go | 7 +- agentplane/notification_service/settings.py | 5 +- .../notification_service/test_settings.py | 5 +- agentplane/runner_gateway/README.md | 4 +- agentplane/runner_gateway/client.py | 15 ++-- agentplane/runner_gateway/config.go | 2 +- agentplane/runner_gateway/config_test.go | 4 +- .../dispatcher/runner_client.go | 9 +- agentplane/runner_gateway/settings.py | 4 +- agentplane/sandbox_controller/README.md | 2 +- agentplane/sandbox_controller/SPEC.md | 5 +- .../sandbox_controller/cmd/config_test.go | 6 +- .../internal/runnercontrol/quiescer.go | 7 +- agentplane/sandbox_controller/settings.py | 4 +- agentplane/sandboxes/api/constants.go | 7 +- .../environmentruntimeclient/client_test.go | 4 +- .../system_tests/deployment/test_render.py | 2 +- .../system_tests/driver/central_sampler.py | 39 ++------- .../system_tests/scenarios/central_sampler.py | 84 +++---------------- cluster/cdk8s/agentplane/runner_gateway.py | 5 +- .../agentplane/runner_gateway_workload.py | 2 +- .../cdk8s/agentplane/sandbox_controller.py | 4 +- .../agentplane-staging-delivery.k8s.yaml | 2 +- .../agentplane-staging.k8s.yaml | 8 +- 38 files changed, 108 insertions(+), 201 deletions(-) diff --git a/agentplane/app/README.md b/agentplane/app/README.md index 61736b362a0..4c096b5a053 100644 --- a/agentplane/app/README.md +++ b/agentplane/app/README.md @@ -671,7 +671,7 @@ snapshot, and changing the Environment incarnation or account discards the prior - **Connection direction:** the Runner Gateway dials the runner's private mTLS gRPC server ([runner control](../docs/runner_control.md)); the runner pushes journals to Event ingress. - Callers use the per-Sandbox control Service across Pod replacements. + Callers use the stable Runner Gateway Service across Pod replacements. - **Model credentials:** staging uses a dedicated key for OpenAI/Claude subscription and local Ollama chat routes; testing uses the `cheap-experiments` LiteLLM key. The Pod holds no key: the sidecar strips caller Authorization on the exact-origin SVID route, the central proxy diff --git a/agentplane/app/internal/bridge/bridge.go b/agentplane/app/internal/bridge/bridge.go index eba70884516..0a5a951854f 100644 --- a/agentplane/app/internal/bridge/bridge.go +++ b/agentplane/app/internal/bridge/bridge.go @@ -1,5 +1,5 @@ // Package bridge adapts the app's named Sandbox and thread contracts to the -// Session Dispatcher and the per-Sandbox Runner Gateway. +// Session Dispatcher and the Runner Gateway's Sandbox proxy. package bridge import ( @@ -186,7 +186,7 @@ func ParseSessionSpec(defaults *sessionpb.SessionSpec, fields map[string]any) (* return merged, nil } -// RunnerConfig configures per-call control Service connections. +// RunnerConfig configures per-call connections to the Runner Gateway's Sandbox proxy. type RunnerConfig struct { TLS *peertls.ClientTLS Namespace string diff --git a/agentplane/app/internal/bridge/runner.go b/agentplane/app/internal/bridge/runner.go index 49b22763212..95e98276711 100644 --- a/agentplane/app/internal/bridge/runner.go +++ b/agentplane/app/internal/bridge/runner.go @@ -11,8 +11,8 @@ import ( "github.com/agentydragon/ducktape/agentplane/sandboxes/environmentruntimeclient" ) -// GatewayRunnerProxy invokes informational Runner RPCs through the named -// Sandbox's control Service and supplies its current identity on every call. +// GatewayRunnerProxy invokes informational Runner RPCs through the stable Runner Gateway Service +// and supplies the named Sandbox's identity on every call. type GatewayRunnerProxy struct{ config RunnerConfig } // NewGatewayRunnerProxy validates the per-Sandbox runner transport settings. @@ -21,7 +21,7 @@ func NewGatewayRunnerProxy(config RunnerConfig) (*GatewayRunnerProxy, error) { return nil, errors.New("runner namespace, token file, and positive timeout are required") } if config.TargetTemplate == "" { - config.TargetTemplate = sandboxapi.ControlServiceTargetTemplate() + config.TargetTemplate = sandboxapi.RunnerGatewayTargetTemplate() } return &GatewayRunnerProxy{config: config}, nil } diff --git a/agentplane/app/internal/settings/config_test.go b/agentplane/app/internal/settings/config_test.go index ad5b96890a2..4049d488c76 100644 --- a/agentplane/app/internal/settings/config_test.go +++ b/agentplane/app/internal/settings/config_test.go @@ -46,7 +46,7 @@ action_federation: scope: actions:operator sandbox_namespace: sandboxes runner_gateway_target: runner-gateway:9090 -runner_gateway_control_target: '{sandbox}-control.{namespace}.svc:8080' +runner_gateway_control_target: 'runner-gateway.{namespace}.svc:8080' session_dispatcher_target: session-dispatcher:9090 history_service_target: history:9090 notifications_url: http://notifications:9090 @@ -81,7 +81,7 @@ func validTestConfig() Config { SandboxNamespace: "sandboxes", RunnerGatewayTarget: "runner-gateway:9090", SessionDispatcherTarget: "session-dispatcher:9090", - RunnerGatewayControlTarget: "{sandbox}-control.{namespace}.svc:8080", + RunnerGatewayControlTarget: "runner-gateway.{namespace}.svc:8080", HistoryServiceTarget: "history:9090", PolicyServiceTarget: "policy-controller:8082", PolicyServiceTokenFile: "/var/run/policy/token", diff --git a/agentplane/app/settings.py b/agentplane/app/settings.py index 3ab887c7242..70b668938c6 100644 --- a/agentplane/app/settings.py +++ b/agentplane/app/settings.py @@ -123,9 +123,8 @@ class Settings(BaseModel): policy_service_token_file: Path = Path("/var/run/secrets/agentplane-policy/token") runner_gateway_control_target: str = Field( min_length=1, - description="Each Sandbox's control Service, a template of `{sandbox}` and `{namespace}` such as " - "`{sandbox}-control.{namespace}.svc:8080`. Lifecycle calls go there, to the replica holding the " - "runner connection.", + description="Stable Runner Gateway Service target, typically `runner-gateway.{namespace}.svc:8080`; " + "the Gateway routes each lifecycle call using the Sandbox identity.", ) history_service_target: str = Field(min_length=1) notifications_url: HttpEndpointUrl = Field( diff --git a/agentplane/central_sampler/README.md b/agentplane/central_sampler/README.md index a47154cef00..b34bef6ec0b 100644 --- a/agentplane/central_sampler/README.md +++ b/agentplane/central_sampler/README.md @@ -107,10 +107,10 @@ sampler's durable intent journal; they do not deduplicate or cache server result retained native session during a Gateway outage, independently of the first-open and typed RPC timeouts. An outage longer than the exec-server's detached-session lifetime can return a definitive native loss response when Gateway recovers; the sampler then records a model notice before later work. -Set `gateway_target` to `{sandbox}-control.{namespace}.svc:8080` for the per-Sandbox control Service: -it selects the Gateway replica holding that Runner's observation Lease. A load-balanced Gateway -Service is unsuitable when there are multiple replicas. Fixed targets support single-process -fixtures. Templates accept DNS-label executor names and only these two placeholders. A fixed +Set `gateway_target` to `runner-gateway.{namespace}.svc:8080` for the stable shared Runner Gateway Service. +Each Gateway replica observes Runtimes independently, so the load-balanced Service works with +multiple replicas. Fixed targets support single-process fixtures. Templates accept DNS-label +executor names and only these two placeholders. A fixed `direct_executor_target` supports standalone fixtures without Gateway only when `direct_executor_insecure_localtest` is explicitly true and the target is a loopback IP address. Choose exactly one Gateway or direct execution target. Production execution uses Gateway bearer diff --git a/agentplane/central_sampler/SPEC.md b/agentplane/central_sampler/SPEC.md index d19d6dfb500..fd2c3a0ec85 100644 --- a/agentplane/central_sampler/SPEC.md +++ b/agentplane/central_sampler/SPEC.md @@ -614,9 +614,9 @@ above; this operator path remains for a blocked Session. See [blocked Session re The sampler uses one generated typed `Executor` client for direct local-test and Gateway targets. The `Executor` service contract terminates at the Sandbox's Runner, which reaches only its configured loopback adapter; Gateway proxies the same service after authorizing the sampler and checking the -Sandbox UID and incarnation. The configured Gateway target selects the per-Sandbox control Service -and thus the Gateway replica holding that Runner's observation Lease. No caller-chosen executor URL -crosses the API. The Runner's adapter holds its local WebSocket on the server-streaming +Sandbox UID and incarnation. The configured Gateway target is the stable `runner-gateway` Service; +each replica routes to its local Runner connection using the Sandbox UID and incarnation. No +caller-chosen executor URL crosses the API. The Runner's adapter holds its local WebSocket on the server-streaming `OpenSession` connection. An active stream blocks executor Sandbox autosuspend until it closes. Dispatcher does not relay tools. diff --git a/agentplane/central_sampler/cmd/main_test.go b/agentplane/central_sampler/cmd/main_test.go index 0d11755a0fb..121f3691d7e 100644 --- a/agentplane/central_sampler/cmd/main_test.go +++ b/agentplane/central_sampler/cmd/main_test.go @@ -78,7 +78,7 @@ func validStartupDiagnosticSettings(t *testing.T) samplersettings.Settings { "code_mode_host": "/usr/local/bin/codex-code-mode-host", "executor_codex": "/usr/local/bin/codex", "logs": "/tmp/sampler-logs", - "gateway_target": "sandbox-control.{namespace}.svc:8080", + "gateway_target": "runner-gateway.{namespace}.svc:8080", "gateway_token_file": "/var/run/secrets/gateway/token", "max_live_sessions": 1, "rpc_timeout_seconds": 1, diff --git a/agentplane/central_sampler/internal/settings/settings_test.go b/agentplane/central_sampler/internal/settings/settings_test.go index f48062959e7..9bbf2a0db58 100644 --- a/agentplane/central_sampler/internal/settings/settings_test.go +++ b/agentplane/central_sampler/internal/settings/settings_test.go @@ -658,10 +658,10 @@ func TestGatewayTemplateCannotInjectAddress(t *testing.T) { s := fixtureSettings() s.DirectExecutorTarget = "" s.DirectExecutorInsecureLocaltest = false - s.GatewayTarget = "{sandbox}-control.{namespace}.svc.cluster.local:8080" + s.GatewayTarget = "runner-gateway.{namespace}.svc.cluster.local:8080" s.GatewayTokenFile = "/fixture/token" address, err := s.GatewayAddress("test", "executor") - if err != nil || address != "executor-control.test.svc.cluster.local:8080" { + if err != nil || address != "runner-gateway.test.svc.cluster.local:8080" { t.Fatalf("address=%q err=%v", address, err) } for _, name := range []string{"bad.example", "UPPER", "", strings.Repeat("a", 64), "../other", "elsewhere:1234/"} { diff --git a/agentplane/docs/runner_control.md b/agentplane/docs/runner_control.md index 67751395225..d21253b9a9c 100644 --- a/agentplane/docs/runner_control.md +++ b/agentplane/docs/runner_control.md @@ -40,7 +40,8 @@ the incarnation. Observation loss removes that replica's callable connection and attempt after one second; it does not revoke another replica's connection. Replacement observers wait for their predecessor to exit before publishing their local connection. -Existing callers still use `-control`. The connection Lease and EndpointSlice temporarily +Callers use the stable `runner-gateway` Service. During their rollout, old callers still use +`-control`. The connection Lease and EndpointSlice temporarily select its destination, independently of Runtime call authority. Each replica attempts route publication after becoming callable; a live competing route claim does not block its own calls. Route metadata renews on Runtime snapshots and heartbeats. A contender takes over after observing @@ -71,8 +72,8 @@ runner journal; a repeated ID with a different operation or payload fails `ALREA ## Runner proxy Every Gateway replica serves `agentplane.env.runtime.control.EnvironmentRuntime` -on its public gRPC port. The Sandbox's `-control` Service temporarily selects one replica -for existing callers. The proxy forwards +on its public gRPC port. Callers reach any replica through the stable `runner-gateway` Service. +The Sandbox's `-control` Service remains available for old caller processes during rollout. The proxy forwards unary calls over the runner gRPC connection and returns the runner's status and response. - **Sandbox.** The caller names it in the `x-agentplane-sandbox` header, the same name its target @@ -239,7 +240,7 @@ may reach any still-current replica, which can serve independently; a stopped ob ## Lifecycle operations The Runner Gateway makes Session calls over the runner gRPC connection. App and Notification callers -reach the temporary selected route through the control Service (`control_target`, a required setting). +reach the stable `runner-gateway` Service (`control_target`, a required setting). - `ListSessions` reads retained Session inventory. - `GetStatus` reads a coherent live snapshot of process incarnations, Session assignments and @@ -257,8 +258,8 @@ reach the temporary selected route through the control Service (`control_target` It waits for preceding admissions; subsequent new-work calls fail `ABORTED`. Final ingress acknowledgement is required for `QUIESCED`. -Any replica with a current independent observation may call it. Existing callers reach a selected -replica through the temporary control Service. Runtime connectivity grants no Dispatcher ownership. Transport reconnect never replays calls; durable command retry uses the +Any replica with a current independent observation may call it. Callers reach any replica through the stable Service; old caller processes retain their +per-Sandbox route until rollout completes. Runtime connectivity grants no Dispatcher ownership. Transport reconnect never replays calls; durable command retry uses the original command ID. Process identities in diagnostics are per-spawn UUIDs, not command addresses. Stored native conversations may be dormant while their store's app server is running. A Session becomes ready after its native conversation preparation/reconciliation, without needing a turn. diff --git a/agentplane/docs/system_overview.md b/agentplane/docs/system_overview.md index 042b149bc4f..8a07f1e2588 100644 --- a/agentplane/docs/system_overview.md +++ b/agentplane/docs/system_overview.md @@ -209,7 +209,8 @@ embeds files through an embedding endpoint and owns its own PostgreSQL database retained history. - **Runner control uses native gRPC.** The Gateway dials each runner's private server using mTLS. It validates the certificate against the active Sandbox/workload and retains `-control` - routing and `-environment-runtime` Leases solely for legacy route publication. + routing and `-environment-runtime` Leases solely for legacy route publication during caller + rollout. Current callers use the stable `runner-gateway` Service. Every replica independently watches and calls the Runtime without a connection epoch. The Sandbox Controller's direct mTLS `observe_only` watch supplies the accepted workload UID and binds it for the Runtime process before the first facts snapshot; the Controller keeps those facts in memory per diff --git a/agentplane/notification_service/README.md b/agentplane/notification_service/README.md index 31397069968..8dd7fc77a50 100644 --- a/agentplane/notification_service/README.md +++ b/agentplane/notification_service/README.md @@ -157,8 +157,8 @@ Configuration changes require a service restart. The URL must be an HTTP(S) origin; Go derives the gRPC host and port from it, using h2c for `http` and TLS for `https`. A peer `tls` block verifies its configured server SPIFFE URI and CA without presenting a client certificate. This block applies independently of the legacy Actions URL scheme. -`runner_gateway` uses `control_target` and `token_file` for per-Sandbox -runner live-state checks. +`runner_gateway` uses `control_target` and `token_file` for runner proxy calls through the stable +Runner Gateway Service. `history_service` and `session_dispatcher` each contain a `target` and `token_file` (`request_timeout_s` is optional for both). The Dispatcher setting is required. Each service checks its own token audience, so each token file holds a token projected for that service. Token paths refer diff --git a/agentplane/notification_service/config.go b/agentplane/notification_service/config.go index 046ae177fe7..2a27548eeb2 100644 --- a/agentplane/notification_service/config.go +++ b/agentplane/notification_service/config.go @@ -62,8 +62,8 @@ type config struct { // runnerGateway is how the service reaches a Sandbox's runner through the Runner Gateway's proxy. type runnerGateway struct { TLS *peertls.ClientTLS `mapstructure:"tls"` - // ControlTarget is each Sandbox's control Service, a template of `{sandbox}` and `{namespace}` such as - // `{sandbox}-control.{namespace}.svc:8080`. + // ControlTarget is the stable Runner Gateway Service, a template of `{namespace}` such as + // `runner-gateway.{namespace}.svc:8080`. ControlTarget string `mapstructure:"control_target"` TokenFile string `mapstructure:"token_file"` RequestTimeoutSeconds float64 `mapstructure:"request_timeout_s"` diff --git a/agentplane/notification_service/config_test.go b/agentplane/notification_service/config_test.go index 57ac447ab2d..91a9c660830 100644 --- a/agentplane/notification_service/config_test.go +++ b/agentplane/notification_service/config_test.go @@ -26,7 +26,7 @@ actions: url: http://test-actions.invalid:8080 token_file: /test/tokens/actions runner_gateway: - control_target: "{sandbox}-control.{namespace}.svc:8080" + control_target: "runner-gateway.{namespace}.svc:8080" token_file: /test/tokens/gateway history_service: target: test-history.invalid:8080 @@ -61,7 +61,7 @@ actions: ca_file: /test/ca.pem server_uri: spiffe://agentplane/actions runner_gateway: - control_target: "{sandbox}-control.{namespace}.svc:8080" + control_target: "runner-gateway.{namespace}.svc:8080" token_file: /test/tokens/gateway tls: ca_file: /test/ca.pem @@ -106,7 +106,7 @@ func TestTheSettingsTheDeploymentManifestsSetLoadWithTheirSecretsFromTheEnvironm if c.DatabaseURL == "" || c.Namespace != "test-agentplane" || c.TokenAudience != "test-notifications" || c.OperatorReaderAccount != "test-app" || (c.GitHubRetention.Days == nil || *c.GitHubRetention.Days != 14) || c.NoticeDebounce.QuietSeconds != 60 || c.NoticeDebounce.MaxWaitSeconds != 120 || c.Actions.URL != "http://test-actions.invalid:8080" || c.Actions.TokenFile != "/test/tokens/actions" || - c.RunnerGateway.ControlTarget != "{sandbox}-control.{namespace}.svc:8080" || c.RunnerGateway.TokenFile != "/test/tokens/gateway" || + c.RunnerGateway.ControlTarget != "runner-gateway.{namespace}.svc:8080" || c.RunnerGateway.TokenFile != "/test/tokens/gateway" || c.HistoryService.Target != "test-history.invalid:8080" || c.HistoryService.TokenFile != "/test/tokens/history" || c.SessionDispatcher.Target != "test-dispatcher.invalid:8080" || c.SessionDispatcher.TokenFile != "/test/tokens/dispatcher" || c.RequestTimeoutSeconds != 15 || c.MessageBodyLimitBytes != 32768 || c.GitHub.APIVersion != "2022-11-28" { @@ -221,7 +221,7 @@ func TestSettingsAreRefusedWithoutEchoingTheirValues(t *testing.T) { "Actions URL credentials": {file: strings.Replace(requiredFile, "http://test-actions.invalid:8080", "http://test-user@actions.invalid:8080", 1)}, "Actions URL invalid port": {file: strings.Replace(requiredFile, "http://test-actions.invalid:8080", "http://test-actions.invalid:70000", 1)}, "no Actions token file": {file: strings.Replace(requiredFile, " token_file: /test/tokens/actions\n", "", 1)}, - "no control target": {file: strings.Replace(requiredFile, " control_target: \"{sandbox}-control.{namespace}.svc:8080\"\n", "", 1)}, + "no control target": {file: strings.Replace(requiredFile, " control_target: \"runner-gateway.{namespace}.svc:8080\"\n", "", 1)}, "no History token file": {file: strings.Replace(requiredFile, " token_file: /test/tokens/history\n", "", 1)}, "no Dispatcher target": {file: strings.Replace(requiredFile, " target: test-dispatcher.invalid:8080\n", "", 1)}, "HTTP port out of range": {file: requiredFile + "http_port: 70000\n"}, diff --git a/agentplane/notification_service/internal/rpc/runner.go b/agentplane/notification_service/internal/rpc/runner.go index 85cb33a068b..3ceaf3f778e 100644 --- a/agentplane/notification_service/internal/rpc/runner.go +++ b/agentplane/notification_service/internal/rpc/runner.go @@ -13,21 +13,20 @@ import ( "github.com/agentydragon/ducktape/agentplane/sandboxes/sandboxreader" ) -// ProxyRunner asks a Sandbox's runner through the Sandbox's control Service, as the Session Dispatcher does: -// every call dials the Service afresh, so it reaches the runner's current connection holder. +// ProxyRunner asks a Sandbox's runner through the stable Runner Gateway Service, as the Session Dispatcher does. type ProxyRunner struct { TLS *peertls.ClientTLS // TokenFile is the projected ServiceAccount token the Runner Gateway proxy admits, read on every call. TokenFile string Timeout time.Duration - // TargetTemplate defaults to the per-Sandbox control Service when empty. A test points it at a fixed server. + // TargetTemplate defaults to the stable Runner Gateway Service when empty. A test points it at a fixed server. TargetTemplate string } func (r *ProxyRunner) ListSessions(ctx context.Context, sandbox *sandboxreader.Identity) ([]string, error) { template := r.TargetTemplate if template == "" { - template = sandboxapi.ControlServiceTargetTemplate() + template = sandboxapi.RunnerGatewayTargetTemplate() } var sessions []string err := environmentruntimeclient.WithClient(ctx, environmentruntimeclient.Config{ diff --git a/agentplane/notification_service/settings.py b/agentplane/notification_service/settings.py index 4d163cc66bc..07acdb6b563 100644 --- a/agentplane/notification_service/settings.py +++ b/agentplane/notification_service/settings.py @@ -55,9 +55,8 @@ class RunnerGatewaySettings(BaseModel): tls: ClientTLS | None = None model_config = ConfigDict(extra="forbid", hide_input_in_errors=True) control_target: str = Field( - description="Each Sandbox's control Service, a template of `{sandbox}` and `{namespace}` such as " - "`{sandbox}-control.{namespace}.svc:8080`. Lifecycle calls go there, to the replica holding the " - "runner connection." + description="Stable Runner Gateway Service target, typically `runner-gateway.{namespace}.svc:8080`; " + "the Gateway routes each lifecycle call using the Sandbox identity." ) request_timeout_s: PositiveSeconds = 5 token_file: Path = Field( diff --git a/agentplane/notification_service/test_settings.py b/agentplane/notification_service/test_settings.py index bb18190d908..fff49ae593c 100644 --- a/agentplane/notification_service/test_settings.py +++ b/agentplane/notification_service/test_settings.py @@ -24,10 +24,7 @@ def static_values() -> dict[str, object]: }, "namespace": "test", "actions": {"url": "http://actions", "token_file": "/tokens/actions"}, - "runner_gateway": { - "control_target": "{sandbox}-control.{namespace}.test:8080", - "token_file": "/tokens/gateway", - }, + "runner_gateway": {"control_target": "runner-gateway.{namespace}.test:8080", "token_file": "/tokens/gateway"}, "history_service": {"target": "history:8080", "token_file": "/tokens/history"}, "session_dispatcher": {"target": "dispatcher:8080", "token_file": "/tokens/dispatcher"}, } diff --git a/agentplane/runner_gateway/README.md b/agentplane/runner_gateway/README.md index b8ac8597b96..e303138510f 100644 --- a/agentplane/runner_gateway/README.md +++ b/agentplane/runner_gateway/README.md @@ -54,8 +54,8 @@ incarnation and boot ID, bound from a Runtime-trusted source; the shared Gateway replica choose or advance an incarnation. Per-method caller authorization remains, and uncertain mutations are never replayed automatically. The Dispatcher delivery Lease is separate and unchanged. All replicas now independently observe and call the Controller-bound Runtime without a wire epoch. -The existing per-Sandbox route remains until callers switch to the stable Service after this Gateway -image is pinned. Its Lease/epoch selects legacy routing only, never Runtime call ownership. +Callers now use the stable Service. The per-Sandbox route remains until all old caller processes +have rolled out. Its Lease/epoch selects legacy routing only, never Runtime call ownership. The Dispatcher role retains its PostgreSQL queue, command IDs, per-Sandbox delivery Leases and database ownership generations. Those delivery Leases are separate from Gateway-to-Runtime diff --git a/agentplane/runner_gateway/client.py b/agentplane/runner_gateway/client.py index 2a63b45e6c2..19a040045bf 100644 --- a/agentplane/runner_gateway/client.py +++ b/agentplane/runner_gateway/client.py @@ -1,4 +1,4 @@ -"""gRPC client for the Runner proxy reached through each Sandbox's control Service.""" +"""gRPC client for the Runner proxy reached through the stable Runner Gateway Service.""" from collections.abc import Awaitable, Callable from pathlib import Path @@ -23,9 +23,8 @@ class SandboxControlClient: - """Calls only the replica holding a Sandbox's runner connection answers, through the Sandbox's - control Service, each on a connection dialed for that call alone: a replica that no longer holds - it refuses with UNAVAILABLE, and the caller's retry resolves the control Service again.""" + """Calls the Runner Gateway Service on a fresh connection for each RPC; the Gateway replica + receiving the call uses the Sandbox identity to select its local runner connection.""" def __init__( self, @@ -36,7 +35,7 @@ def __init__( channel_credentials: grpc.ChannelCredentials | None = None, server_name: str | None = None, ) -> None: - # A template of `{sandbox}` and `{namespace}`: `{sandbox}-control.{namespace}.svc:8080`. + # A template of `{namespace}`: `runner-gateway.{namespace}.svc:8080`. self.control_target = control_target self.token_file = token_file self.request_timeout_s = request_timeout_s @@ -55,7 +54,7 @@ async def runner_unary[T]( timeout_s: float | None = None, expected_incarnation_uid: str | None = None, ) -> T: - """Call a Runner RPC through its control Service, fenced to the expected Sandbox identity.""" + """Call a Runner RPC through the shared Gateway Service, fenced to the Sandbox identity.""" incarnation_metadata = ( ((INCARNATION_UID_HEADER, expected_incarnation_uid),) if expected_incarnation_uid is not None else () ) @@ -85,7 +84,7 @@ async def runner_unary[T]( async def list_sessions( self, destination: SandboxDestination, *, expected_incarnation_uid: str | None = None ) -> list[session_pb2.SessionSummary]: - """Read live runner inventory through its proxy, fenced to the expected Sandbox identity.""" + """Read live runner inventory through the Gateway proxy, fenced to the expected Sandbox identity.""" response = await self.runner_unary( destination, lambda stub: stub.ListSessions, @@ -96,7 +95,7 @@ async def list_sessions( class SandboxRunner: - """One Sandbox's runner, reached through the replica holding its connection.""" + """One Sandbox's runner, reached through the shared Runner Gateway Service.""" def __init__(self, control: SandboxControlClient, destination: SandboxDestination) -> None: self.control = control diff --git a/agentplane/runner_gateway/config.go b/agentplane/runner_gateway/config.go index b035da7cb7f..7bd9f730d18 100644 --- a/agentplane/runner_gateway/config.go +++ b/agentplane/runner_gateway/config.go @@ -99,7 +99,7 @@ type delegationConfig struct { func defaultConfig() config { return config{ ProxyPort: sandboxapi.ControlServicePort, - RunnerControlTargetTemplate: sandboxapi.ControlServiceTargetTemplate(), + RunnerControlTargetTemplate: sandboxapi.RunnerGatewayTargetTemplate(), QueueCap: 1000, WatchLeaseSeconds: 300, RunnerTimeoutSeconds: 10, } } diff --git a/agentplane/runner_gateway/config_test.go b/agentplane/runner_gateway/config_test.go index 6983208154e..0117f66c1e2 100644 --- a/agentplane/runner_gateway/config_test.go +++ b/agentplane/runner_gateway/config_test.go @@ -41,7 +41,7 @@ gateway_audience: runner-gateway dispatcher_callers: [agentplane/app, agentplane/notifications] gateway_callers: [agentplane/runner-gateway] sandbox_controller_account: agentplane/sandbox-controller -runner_control_target_template: '{sandbox}-control.{namespace}.svc:8080' +runner_control_target_template: 'runner-gateway.{namespace}.svc:8080' runner_security: {mode: mtls, tls: {cert_file: /tmp/test-cert, key_file: /tmp/test-key, ca_file: /tmp/test-ca}} gateway_tls: {ca_file: /tmp/workload-ca, server_uri: spiffe://agentplane/session-dispatcher} rpc: @@ -360,7 +360,7 @@ func TestEmptyWakeCallerListIsAccepted(t *testing.T) { func TestHostProcessTargetsAreAcceptedAndTheAdvertisedPortIsBounded(t *testing.T) { settings := strings.Replace(requiredFile, "listen_address: ':8080'", "listen_address: 127.0.0.1:41001", 1) - settings = strings.Replace(settings, "runner_control_target_template: '{sandbox}-control.{namespace}.svc:8080'", + settings = strings.Replace(settings, "runner_control_target_template: 'runner-gateway.{namespace}.svc:8080'", "runner_control_target_template: 127.0.0.1:41001", 1) + ` proxy_port: 42000 kubeconfig: /tmp/fake-kubeconfig diff --git a/agentplane/runner_gateway/dispatcher/runner_client.go b/agentplane/runner_gateway/dispatcher/runner_client.go index fd872d38876..d7cd6839d9f 100644 --- a/agentplane/runner_gateway/dispatcher/runner_client.go +++ b/agentplane/runner_gateway/dispatcher/runner_client.go @@ -14,16 +14,15 @@ import ( "github.com/agentydragon/ducktape/agentplane/sandboxes/environmentruntimeclient" ) -// ProxyRunner dials the per-Sandbox control Service for each call. A new -// connection resolves the holder replica again after a Lease handoff; command -// IDs reconcile uncertain replies at the runner journal. +// ProxyRunner dials the stable Runner Gateway Service for each call. Any replica can +// serve the call, and command IDs reconcile uncertain replies at the runner journal. type ProxyRunner struct { Namespace string TokenFile string Timeout time.Duration // GatewayTLS pins the Session Dispatcher server SVID without presenting a client certificate. GatewayTLS *peertls.ClientTLS - // Defaults to the per-Sandbox control Service when empty. Templates can also + // Defaults to the stable Runner Gateway Service when empty. Templates can also // be a fixed target for host-process integration tests. TargetTemplate string } @@ -31,7 +30,7 @@ type ProxyRunner struct { func (r *ProxyRunner) target(sandbox string) string { template := r.TargetTemplate if template == "" { - template = sandboxapi.ControlServiceTargetTemplate() + template = sandboxapi.RunnerGatewayTargetTemplate() } return environmentruntimeclient.ExpandTarget(template, sandbox, r.Namespace) } diff --git a/agentplane/runner_gateway/settings.py b/agentplane/runner_gateway/settings.py index c57b2d14e65..7a8c1cd17f9 100644 --- a/agentplane/runner_gateway/settings.py +++ b/agentplane/runner_gateway/settings.py @@ -77,9 +77,9 @@ class Settings(BaseModel): default=8080, ge=1, le=65535, description="Port advertised in each Sandbox control EndpointSlice." ) runner_control_target_template: str = Field( - default="{sandbox}-control.{namespace}.svc:8080", + default="runner-gateway.{namespace}.svc:8080", min_length=1, - description="Target or target template the Dispatcher uses for each Sandbox runner proxy.", + description="Stable Runner Gateway Service target or template used for each Sandbox runner proxy.", ) dispatcher_audience: str = Field(min_length=1, description="Dispatcher caller TokenReview audience.") gateway_audience: str = Field(min_length=1, description="Gateway caller TokenReview audience.") diff --git a/agentplane/sandbox_controller/README.md b/agentplane/sandbox_controller/README.md index a623396ec9c..78adee7b516 100644 --- a/agentplane/sandbox_controller/README.md +++ b/agentplane/sandbox_controller/README.md @@ -13,7 +13,7 @@ through the Sandbox's launch ConfigMap (a Pod mounts it, a VM gets it as a disk) defaults live in the app's preset catalog and are resolved before an opening reaches the Dispatcher. Deletion first has the runner seal every Session: the controller calls `Runner.Quiesce` through the [Runner Gateway](../runner_gateway/README.md)'s -[runner proxy](../docs/runner_control.md#runner-proxy), at the Sandbox's own control Service, and sets the +[runner proxy](../docs/runner_control.md#runner-proxy), at the stable Runner Gateway Service, and sets the `Quiesced` condition. Suspension seals the same way before the Pod stops, recorded as `status.quiescedIncarnation`. `runner_security` selects the Controller's own direct observer credentials; `runner_gateway_target` and `runner_gateway_token_file` remain required settings in the mounted YAML file. diff --git a/agentplane/sandbox_controller/SPEC.md b/agentplane/sandbox_controller/SPEC.md index e0decfca71c..1f64103370a 100644 --- a/agentplane/sandbox_controller/SPEC.md +++ b/agentplane/sandbox_controller/SPEC.md @@ -251,8 +251,9 @@ Gateway token; token contents remain in the rotating read-only projected volume. ## Control Service Every Sandbox has a headless, selector-less `-control` Service on the Runner Gateway's -public proxy port, 8080. Callers use their workload bearers to reach the connection holder through -this Service. The Gateway connects directly to the current workload's mTLS runner listener on port 9090. The controller repairs the per-Sandbox Service's managed fields; the Gateway owns its EndpointSlice. +public proxy port, 8080. Callers use their workload bearers to reach any replica through +the stable Runner Gateway Service. The per-Sandbox Service remains available for old caller processes +during rollout. The Gateway connects directly to the current workload's mTLS runner listener on port 9090. The controller repairs the per-Sandbox Service's managed fields; the Gateway owns its EndpointSlice. A foreign same-name Service blocks new workload starts. ## Launch diff --git a/agentplane/sandbox_controller/cmd/config_test.go b/agentplane/sandbox_controller/cmd/config_test.go index 1f639e342eb..af6a354e9b0 100644 --- a/agentplane/sandbox_controller/cmd/config_test.go +++ b/agentplane/sandbox_controller/cmd/config_test.go @@ -13,7 +13,7 @@ import ( const validSettings = `namespace: agentplane metrics_bind_address: ":9090" health_probe_bind_address: ":9091" -runner_gateway_target: "{sandbox}-control.{namespace}.svc:8080" +runner_gateway_target: "runner-gateway.{namespace}.svc:8080" runner_gateway_token_file: /var/run/secrets/agentplane-runner-gateway/token runner_security: mode: mtls @@ -51,7 +51,7 @@ func TestLoadConfigReadsYAMLAndIgnoresPerSettingEnvironment(t *testing.T) { } want := config{ Namespace: "agentplane", MetricsBindAddress: ":9090", HealthProbeBindAddress: ":9091", - RunnerGatewayTarget: "{sandbox}-control.{namespace}.svc:8080", + RunnerGatewayTarget: "runner-gateway.{namespace}.svc:8080", RunnerGatewayTokenFile: "/var/run/secrets/agentplane-runner-gateway/token", RunnerSecurity: peertls.Security{Mode: peertls.MutualTLS, TLS: &peertls.Config{ CertFile: "/run/agentplane/sandbox-controller-client-tls/tls.crt", @@ -95,7 +95,7 @@ func TestBindAddressesDefaultWhenOmittedFromYAML(t *testing.T) { func TestLoadConfigRejectsInvalidSettingsFiles(t *testing.T) { missing := map[string]string{ "namespace": "namespace: agentplane\n", - "runner_gateway_target": "runner_gateway_target: \"{sandbox}-control.{namespace}.svc:8080\"\n", + "runner_gateway_target": "runner_gateway_target: \"runner-gateway.{namespace}.svc:8080\"\n", "runner_gateway_token_file": "runner_gateway_token_file: /var/run/secrets/agentplane-runner-gateway/token\n", "runner_security": "runner_security:\n mode: mtls\n tls:\n cert_file: /run/agentplane/sandbox-controller-client-tls/tls.crt\n key_file: /run/agentplane/sandbox-controller-client-tls/tls.key\n ca_file: /run/agentplane/workload-trust/ca.crt\n", } diff --git a/agentplane/sandbox_controller/internal/runnercontrol/quiescer.go b/agentplane/sandbox_controller/internal/runnercontrol/quiescer.go index d66ea3dea0d..64f57814cbc 100644 --- a/agentplane/sandbox_controller/internal/runnercontrol/quiescer.go +++ b/agentplane/sandbox_controller/internal/runnercontrol/quiescer.go @@ -25,12 +25,11 @@ import ( // quiesceTimeout bounds the whole idempotent handshake, including route-readiness retries. const quiesceTimeout = 30 * time.Second -// GatewayQuiescer calls `Runner.Quiesce` through the Runner Gateway's proxy, at the Sandbox's own -// control Service, which names the replica holding its runner connection. +// GatewayQuiescer calls `Runner.Quiesce` through the stable Runner Gateway Service. type GatewayQuiescer struct { TLS *peertls.ClientTLS // Target is a template of `{sandbox}` and `{namespace}`, typically - // sandboxapi.ControlServiceTargetTemplate. + // sandboxapi.RunnerGatewayTargetTemplate. Target string // TokenFile is the projected ServiceAccount token the gateway accepts from this controller. TokenFile string @@ -52,7 +51,7 @@ func (g GatewayQuiescer) Quiesce(ctx context.Context, sandbox *sandboxapi.Sandbo defer cancel() err := retry.Do(ctx, retry.WithMaxRetries(2, retry.NewConstant(time.Second)), func(ctx context.Context) error { // Restore-time WatchState can precede Gateway Pod readiness. A fresh connection - // re-resolves the elected control Service, while the expected UID fences every try. + // resolves the shared Service, while the expected UID fences every try. err := environmentruntimeclient.WithClient(ctx, environmentruntimeclient.Config{ Target: target, TokenFile: g.TokenFile, TLS: g.TLS, Timeout: quiesceTimeout, SandboxName: sandbox.Name, SandboxUID: string(sandbox.UID), SandboxNamespace: sandbox.Namespace, diff --git a/agentplane/sandbox_controller/settings.py b/agentplane/sandbox_controller/settings.py index 187d1afb1fa..5993c2b1c4e 100644 --- a/agentplane/sandbox_controller/settings.py +++ b/agentplane/sandbox_controller/settings.py @@ -28,8 +28,8 @@ class Settings(BaseModel): ) runner_gateway_target: str = Field( min_length=1, - description="Where the controller quiesces a Sandbox's runner through the Runner Gateway: one target, or a " - "template of `{sandbox}` and `{namespace}` such as the Sandbox's own control Service.", + description="Where the controller quiesces a Sandbox's runner through the stable Runner Gateway Service: " + "one target, or a template such as `runner-gateway.{namespace}.svc:8080`.", ) runner_gateway_token_file: Path = Field( description="Projected ServiceAccount token the Runner Gateway accepts from this controller." diff --git a/agentplane/sandboxes/api/constants.go b/agentplane/sandboxes/api/constants.go index 25f9a80446c..02bd5bfa319 100644 --- a/agentplane/sandboxes/api/constants.go +++ b/agentplane/sandboxes/api/constants.go @@ -30,12 +30,13 @@ const ( ControlServiceSuffix = "-control" ControlServicePortName = "grpc" ControlServicePort = 8080 + RunnerGatewayService = "runner-gateway" // EnvironmentRuntimeServicePort is the runner's mTLS listener port in a Pod or KubeVirt guest. EnvironmentRuntimeServicePort = 9090 EnvironmentRuntimeCredentialMetadataKey = "agentplane.allegedly.works/environment-runtime-credential" ) -// ControlServiceTargetTemplate is the default target used by both the controller and dispatcher. -func ControlServiceTargetTemplate() string { - return fmt.Sprintf("{sandbox}%s.{namespace}.svc:%d", ControlServiceSuffix, ControlServicePort) +// RunnerGatewayTargetTemplate routes every Sandbox's Runner RPCs through the stable shared Service. +func RunnerGatewayTargetTemplate() string { + return fmt.Sprintf("%s.{namespace}.svc:%d", RunnerGatewayService, ControlServicePort) } diff --git a/agentplane/sandboxes/environmentruntimeclient/client_test.go b/agentplane/sandboxes/environmentruntimeclient/client_test.go index 6605371c51c..44b898a2920 100644 --- a/agentplane/sandboxes/environmentruntimeclient/client_test.go +++ b/agentplane/sandboxes/environmentruntimeclient/client_test.go @@ -124,8 +124,8 @@ func TestReplaceOutgoingMetadataDropsTheCallersMetadata(t *testing.T) { } func TestExpandTarget(t *testing.T) { - template := sandboxapi.ControlServiceTargetTemplate() - if got := ExpandTarget(template, "one", "agentplane-staging"); got != "one-control.agentplane-staging.svc:8080" { + template := sandboxapi.RunnerGatewayTargetTemplate() + if got := ExpandTarget(template, "one", "agentplane-staging"); got != "runner-gateway.agentplane-staging.svc:8080" { t.Fatalf("target = %q", got) } } diff --git a/agentplane/system_tests/deployment/test_render.py b/agentplane/system_tests/deployment/test_render.py index 2b0de3a1792..8bc1b6a36f3 100644 --- a/agentplane/system_tests/deployment/test_render.py +++ b/agentplane/system_tests/deployment/test_render.py @@ -326,7 +326,7 @@ def test_direct_fixture_uses_gateway_and_embedded_migrations(runfiles: Runfiles) endpoint = f"{dispatcher['metadata']['name']}.{NAMESPACE}.svc.cluster.local:{dispatcher_port}" app_config = yaml.safe_load(_resource(docs, "ConfigMap", "app-config")["data"]["config.yaml"]) assert app_config["session_dispatcher_target"] == endpoint - assert app_config["runner_gateway_control_target"] == "{sandbox}-control.{namespace}.svc:8080" + assert app_config["runner_gateway_control_target"] == "runner-gateway.{namespace}.svc:8080" app_container = _resource(docs, "Deployment", "app")["spec"]["template"]["spec"]["containers"][0] assert app_container.get("args", []) == [] app_env = {entry["name"]: entry for entry in app_container["env"]} diff --git a/agentplane/system_tests/driver/central_sampler.py b/agentplane/system_tests/driver/central_sampler.py index e990b28dd4e..3b26adf094d 100644 --- a/agentplane/system_tests/driver/central_sampler.py +++ b/agentplane/system_tests/driver/central_sampler.py @@ -44,7 +44,7 @@ class CentralStack: sandbox: str sandbox_uid: str incarnation_uid: str - control_target: str + gateway_target: str sampler_pod_name: str runner: SandboxRunner @@ -89,23 +89,6 @@ async def _address(cluster: Cluster, core: k8s.CoreV1Api, service: str, port: in return f"{endpoint.spec.cluster_ip}:{port}" -async def _control_address(cluster: Cluster, connection: k8s.ApiClient, sandbox: str, uid: str) -> str: - # The control Service is headless and selector-less. Its Gateway-owned slice names - # the Runner observation Lease holder, including after the scenario's Gateway rollout. - slices = await k8s.DiscoveryV1Api(connection).list_namespaced_endpoint_slice( - cluster.namespace, label_selector=f"kubernetes.io/service-name={sandbox}-control" - ) - (holder,) = slices.items - assert holder.metadata.labels["endpointslice.kubernetes.io/managed-by"] == "agentplane-runner-gateway" - assert any(owner.uid == uid for owner in holder.metadata.owner_references) - (endpoint,) = holder.endpoints - assert endpoint.conditions.ready - (address,) = endpoint.addresses - (port,) = holder.ports - assert port.protocol == "TCP" - return f"{address}:{port.port}" - - async def _delete_sandbox(cluster: Cluster, name: str, uid: str) -> None: with tracer.start_as_current_span( "system_test.central_sampler.sandbox_cleanup", record_exception=False, set_status_on_exception=False @@ -253,26 +236,16 @@ async def central_stack(cluster: Cluster, directory: Path, *, template: str) -> sampler_pod = await _pod(cluster, core, SERVICE) stack.push_async_callback(capture_sampler_diagnostics, cluster, core) gateway_pod = await _pod(cluster, core, "runner-gateway") - # Controller readiness does not wait for legacy Gateway route publication. - # Wait for creation and routing within the same existing 60-second budget. - async with asyncio.timeout(60): - await cluster.kubectl("wait", f"endpointslice/{sandbox}-control", "--for=create", "--timeout=60s") - await cluster.kubectl( - "wait", - f"endpointslice/{sandbox}-control", - f"--for=jsonpath={{.endpoints[0].addresses[0]}}={gateway_pod.status.pod_ip}", - "--timeout=60s", - ) - # The per-Sandbox Service selects the replica holding this Sandbox's Runner stream. - control_target = await _control_address(cluster, connection, sandbox, uid) + # Call through the stable Service so Kubernetes can send every request to any replica. + gateway_target = await _address(cluster, core, "runner-gateway", 8080) sampler_token = await _bound_token(cluster, core, "runner-gateway", gateway_pod, SERVICE) server_credentials = await service_credentials(cluster) - # Sandbox Ready proves Controller observation. A claimed EndpointSlice can precede + # Sandbox Ready proves Controller observation. Service endpoint readiness can precede # the Gateway's first snapshot, so prove the live read route before starting setup. gateway_token = directory / "gateway-reader-token" gateway_token.write_text(await _bound_token(cluster, core, "app", app, GATEWAY_AUDIENCE)) gateway = SandboxControlClient( - control_target=control_target, + control_target=gateway_target, token_file=gateway_token, request_timeout_s=2, channel_credentials=server_credentials, @@ -309,7 +282,7 @@ async def central_stack(cluster: Cluster, directory: Path, *, template: str) -> sandbox=sandbox, sandbox_uid=uid, incarnation_uid=incarnation, - control_target=control_target, + gateway_target=gateway_target, sampler_pod_name=sampler_pod.metadata.name, runner=gateway.runner(destination), ) diff --git a/agentplane/system_tests/scenarios/central_sampler.py b/agentplane/system_tests/scenarios/central_sampler.py index 32379c4d321..0e366d3d859 100644 --- a/agentplane/system_tests/scenarios/central_sampler.py +++ b/agentplane/system_tests/scenarios/central_sampler.py @@ -30,11 +30,7 @@ ) from agentplane.system_tests.driver.cluster import Cluster from agentplane.system_tests.driver.executor import Executor -from agentplane.system_tests.driver.sandbox import ( - ENVIRONMENT_RUNTIME_LEASE_SUFFIX, - SANDBOX_READY_SECONDS, - wait_until_running, -) +from agentplane.system_tests.driver.sandbox import SANDBOX_READY_SECONDS, wait_until_running from agentplane.system_tests.driver.scripted_upstream import ScriptedUpstream from agentplane.system_tests.driver.tracing import tracer from agentplane.testing.codex_model import CodexModel @@ -100,13 +96,6 @@ async def _capture_session_diagnostics( } except Exception as error: diagnostic["sandbox_error"] = repr(error) - try: - environment_runtime_lease = await cluster.kubectl( - "get", "lease", f"{deployed.sandbox}{ENVIRONMENT_RUNTIME_LEASE_SUFFIX}", "-o", "json" - ) - diagnostic["environment_runtime_lease"] = json.loads(environment_runtime_lease) - except Exception as error: - diagnostic["environment_runtime_lease_error"] = repr(error) try: diagnostic["cluster_services_endpoints_pods"] = json.loads( await cluster.kubectl("get", "services,endpointslices,pods", "-o", "json") @@ -666,22 +655,13 @@ async def _roll_gateway_for_central_session( old_uid = old_pod["metadata"]["uid"] killed_native: tuple[str, str, str] | None = None read_failure = None - old_lease = None if abrupt: - old_lease = json.loads( - await cluster.kubectl("get", "lease", f"{deployed.sandbox}{ENVIRONMENT_RUNTIME_LEASE_SUFFIX}", "-o", "json") - ) - assert old_lease["spec"]["holderIdentity"] == old_pod["metadata"]["name"], old_lease assert crictl_command is not None try: if abrupt: assert crictl_command is not None - (undeclared_outputs_dir() / "gateway-crash-before.json").write_text( - json.dumps({"pod": old_pod, "lease": old_lease}, indent=2) + "\n" - ) - # Killing first lets kubelet restart this Pod before scale-down; the restarted - # Gateway can claim a new epoch and gracefully release it on SIGTERM. SIGSTOP - # prevents cleanup while deletion removes this Pod from kubelet's desired set. + (undeclared_outputs_dir() / "gateway-crash-before.json").write_text(json.dumps(old_pod, indent=2) + "\n") + # SIGSTOP prevents cleanup while deletion removes this Pod from kubelet's desired set. async with _stopped_gateway_process(cluster, old_pod, crictl_command): await cluster.kubectl("scale", "deployment/runner-gateway", "--replicas=0") await cluster.kubectl( @@ -690,39 +670,15 @@ async def _roll_gateway_for_central_session( "--for=jsonpath={.metadata.deletionTimestamp}", "--timeout=15s", ) + terminating_pod = json.loads( + await cluster.kubectl("get", "pod", old_pod["metadata"]["name"], "-o", "json") + ) (undeclared_outputs_dir() / "gateway-crash-terminating.json").write_text( - json.dumps( - { - "pod": json.loads( - await cluster.kubectl("get", "pod", old_pod["metadata"]["name"], "-o", "json") - ), - "lease": json.loads( - await cluster.kubectl( - "get", - "lease", - f"{deployed.sandbox}{ENVIRONMENT_RUNTIME_LEASE_SUFFIX}", - "-o", - "json", - ) - ), - }, - indent=2, - ) - + "\n" + json.dumps(terminating_pod, indent=2) + "\n" ) else: await cluster.kubectl("scale", "deployment/runner-gateway", "--replicas=0") await cluster.kubectl("wait", f"pod/{old_pod['metadata']['name']}", "--for=delete", "--timeout=45s") - if old_lease is not None: - lease = json.loads( - await cluster.kubectl( - "get", "lease", f"{deployed.sandbox}{ENVIRONMENT_RUNTIME_LEASE_SUFFIX}", "-o", "json" - ) - ) - (undeclared_outputs_dir() / "gateway-crash-deleted.json").write_text(json.dumps(lease, indent=2) + "\n") - assert lease["metadata"]["uid"] == old_lease["metadata"]["uid"], lease - assert lease["spec"]["holderIdentity"] == old_lease["spec"]["holderIdentity"], lease - assert lease["spec"]["leaseTransitions"] == old_lease["spec"]["leaseTransitions"], lease gateway_slices = json.loads( await cluster.kubectl( "get", "endpointslices", "-l", "kubernetes.io/service-name=runner-gateway", "-o", "json" @@ -777,12 +733,7 @@ async def _roll_gateway_for_central_session( while True: slices = json.loads( await cluster.kubectl( - "get", - "endpointslices", - "-l", - f"kubernetes.io/service-name={deployed.sandbox}-control", - "-o", - "json", + "get", "endpointslices", "-l", "kubernetes.io/service-name=runner-gateway", "-o", "json" ) )["items"] if any( @@ -794,11 +745,6 @@ async def _roll_gateway_for_central_session( outage_seconds = asyncio.get_running_loop().time() - unavailable_started break await asyncio.sleep(0.2) - if old_lease is not None: - lease = json.loads( - await cluster.kubectl("get", "lease", f"{deployed.sandbox}{ENVIRONMENT_RUNTIME_LEASE_SUFFIX}", "-o", "json") - ) - assert lease["spec"]["leaseTransitions"] > old_lease["spec"]["leaseTransitions"], lease return ( await sampler_caller_after_gateway_roll(cluster, deployed), outage_seconds, @@ -1708,7 +1654,7 @@ async def dispatcher_central_setup_tool_and_retained_answer( "incarnation_uid": deployed.incarnation_uid, "template": template, "tool_output": output, - "control_target": deployed.control_target, + "gateway_target": deployed.gateway_target, "source_id": terminal.origin.source_id, "terminal_cursor": terminal.cursor, "folded_through": summary.folded_through, @@ -2199,15 +2145,9 @@ async def _assert_central_components_unchanged( ) -> None: for component, expected in pod_uids.items(): assert await _ready_pod_uids(cluster, component) == expected, f"{component} changed during History outage" - control = json.loads(await cluster.kubectl("get", "endpointslice", f"{deployed.sandbox}-control", "-o", "json")) - ready = [ - address - for endpoint in control.get("endpoints") or [] - if (endpoint.get("conditions") or {}).get("ready") is True - for address in endpoint.get("addresses") or [] - ] - assert ready, f"Sandbox runner lost its ready Gateway endpoint: {control}" - assert any(address in deployed.control_target for address in ready), ready + ready = await _ready_service_endpoint_addresses(cluster, "runner-gateway") + assert ready, "Runner Gateway Service lost all ready endpoints during the History outage" + await deployed.runner.list_sessions(expected_incarnation_uid=deployed.incarnation_uid) async def _wait_for_history_cursors( diff --git a/cluster/cdk8s/agentplane/runner_gateway.py b/cluster/cdk8s/agentplane/runner_gateway.py index 309e110920a..cb0d20dd87e 100644 --- a/cluster/cdk8s/agentplane/runner_gateway.py +++ b/cluster/cdk8s/agentplane/runner_gateway.py @@ -21,6 +21,5 @@ def target(namespace: str) -> str: def control_target(namespace: str) -> str: - """Each Sandbox's control Service, `-control`, which names the replica holding its runner - connection, as the `{sandbox}`/`{namespace}` template the clients' `control_target` takes.""" - return f"{{sandbox}}-control.{{namespace}}.svc:{service(namespace).pod_port}" + """Stable shared Runner Gateway Service target, parameterized by the Sandbox namespace.""" + return f"{NAME}.{{namespace}}.svc:{service(namespace).pod_port}" diff --git a/cluster/cdk8s/agentplane/runner_gateway_workload.py b/cluster/cdk8s/agentplane/runner_gateway_workload.py index 437b413b984..12681bd1e98 100644 --- a/cluster/cdk8s/agentplane/runner_gateway_workload.py +++ b/cluster/cdk8s/agentplane/runner_gateway_workload.py @@ -164,7 +164,7 @@ def __init__( proxy = runner_gateway.service(namespace) own_account = ServiceAccountRef(namespace=namespace, name=runner_gateway.NAME) # A delivery worker may run on a different replica from the holder. It calls - # the per-Sandbox control Service with this projected Gateway-audience token. + # the stable Runner Gateway Service with this projected Gateway-audience token. proxy_callers = {**gateway_callers, own_account: proxy.pods} self.proxy_callers = proxy_callers dispatcher_accounts = frozenset(map(_account, dispatcher_callers)) diff --git a/cluster/cdk8s/agentplane/sandbox_controller.py b/cluster/cdk8s/agentplane/sandbox_controller.py index a1b060f5c08..8dbd15fb519 100644 --- a/cluster/cdk8s/agentplane/sandbox_controller.py +++ b/cluster/cdk8s/agentplane/sandbox_controller.py @@ -65,8 +65,8 @@ def __init__( memory_request: Size = _DEFAULT_MEMORY_REQUEST, memory_limit: Size | None = _DEFAULT_MEMORY_LIMIT, ) -> None: - """The controller quiesces runners through the Runner Gateway's proxy, at each Sandbox's own - control Service (`gateway_control_target`) under a token of `gateway_audience`.""" + """The controller quiesces runners through the stable Runner Gateway Service under a token + of `gateway_audience`.""" super().__init__(scope, id) self.env = env self.gateway = gateway diff --git a/cluster/generated/agentplane-staging-delivery/agentplane-staging-delivery.k8s.yaml b/cluster/generated/agentplane-staging-delivery/agentplane-staging-delivery.k8s.yaml index f1821b5fe02..9ac4b282b2c 100644 --- a/cluster/generated/agentplane-staging-delivery/agentplane-staging-delivery.k8s.yaml +++ b/cluster/generated/agentplane-staging-delivery/agentplane-staging-delivery.k8s.yaml @@ -343,7 +343,7 @@ data: ca_file: /run/agentplane/workload-trust/ca-certificates.crt server_uri: spiffe://agentplane/session-dispatcher proxy_port: 8080 - runner_control_target_template: "{sandbox}-control.{namespace}.svc:8080" + runner_control_target_template: runner-gateway.{namespace}.svc:8080 dispatcher_audience: session-dispatcher gateway_audience: agentplane-runner-gateway runner_security: diff --git a/cluster/k8s/agentplane-staging/agentplane-staging.k8s.yaml b/cluster/k8s/agentplane-staging/agentplane-staging.k8s.yaml index be2d428cf93..2cf5889698a 100644 --- a/cluster/k8s/agentplane-staging/agentplane-staging.k8s.yaml +++ b/cluster/k8s/agentplane-staging/agentplane-staging.k8s.yaml @@ -3440,7 +3440,7 @@ data: "logs": "/var/log/central-sampler", "history_target": "history-service.agentplane-staging.svc.cluster.local:8080", "history_token_file": "/var/run/secrets/central-sampler/history", - "gateway_target": "{sandbox}-control.{namespace}.svc:8080", + "gateway_target": "runner-gateway.{namespace}.svc:8080", "gateway_token_file": "/var/run/secrets/central-sampler/gateway", "direct_executor_insecure_localtest": false, "max_live_sessions": 32, @@ -4315,7 +4315,7 @@ data: session_dispatcher_target: session-dispatcher.agentplane-staging.svc.cluster.local:8080 policy_service_target: policy-controller.agentplane-staging.svc.cluster.local:8082 policy_service_token_file: /var/run/secrets/agentplane-policy/token - runner_gateway_control_target: "{sandbox}-control.{namespace}.svc:8080" + runner_gateway_control_target: runner-gateway.{namespace}.svc:8080 history_service_target: history-service.agentplane-staging.svc.cluster.local:8080 notifications_url: http://notifications.agentplane-staging.svc:8080/ notifications_token_file: /var/run/secrets/agentplane-notifications/token @@ -6631,7 +6631,7 @@ data: namespace: agentplane-staging metrics_bind_address: :8080 health_probe_bind_address: :8081 - runner_gateway_target: "{sandbox}-control.{namespace}.svc:8080" + runner_gateway_target: runner-gateway.{namespace}.svc:8080 runner_gateway_token_file: /var/run/secrets/agentplane-runner-gateway/token immutable: false --- @@ -8993,7 +8993,7 @@ data: tls: ca_file: /run/agentplane/workload-trust/ca-certificates.crt server_uri: spiffe://agentplane/session-dispatcher - control_target: "{sandbox}-control.{namespace}.svc:8080" + control_target: runner-gateway.{namespace}.svc:8080 request_timeout_s: 5 token_file: /var/run/secrets/notifications/gateway session_dispatcher: From e3e2078b0d63130686f699c6061aadd75d746fd0 Mon Sep 17 00:00:00 2001 From: agentydragon-agent <260791330+agentydragon-agent@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:09:01 -0800 Subject: [PATCH 2/5] Follow the yielded recovery probe before checking native session continuity --- .../system_tests/scenarios/central_sampler.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/agentplane/system_tests/scenarios/central_sampler.py b/agentplane/system_tests/scenarios/central_sampler.py index 0e366d3d859..e58b38030c8 100644 --- a/agentplane/system_tests/scenarios/central_sampler.py +++ b/agentplane/system_tests/scenarios/central_sampler.py @@ -393,7 +393,7 @@ async def _wait_for_executor_dispatch_after( async def _follow_code_mode_cell( model: CodexModel, request: ModelRequest[ResponsesRequest], call_id: str, *, timeout_s: float -) -> tuple[ModelRequest[ResponsesRequest], str, str]: +) -> tuple[ModelRequest[ResponsesRequest], str, str, ToolOutput]: """Wait for the same running code-mode cell; never reissue its executor operation.""" output_blocks: list[str] = [] visible_blocks: list[str] = [] @@ -416,7 +416,7 @@ async def _follow_code_mode_cell( ) running = re.search(r"^Script running with cell ID (\S+)", "\n".join(output.blocks), re.MULTILINE) if running is None: - return request, "\n".join(output_blocks), "\n".join(visible_blocks) + return request, "\n".join(output_blocks), "\n".join(visible_blocks), output waits += 1 wait_call_id = f"{call_id}-wait-{waits}" async with request._exchange as exchange: @@ -714,7 +714,10 @@ async def _roll_gateway_for_central_session( assert failed_resume_after is not None session_id, initial_cursor, read_call_id = failed_resume_after request = await asyncio.wait_for(wait_for_read_failure.request(), timeout=45) - read_failure = await _follow_code_mode_cell(wait_for_read_failure, request, read_call_id, timeout_s=180) + request, read_text, visible, _ = await _follow_code_mode_cell( + wait_for_read_failure, request, read_call_id, timeout_s=180 + ) + read_failure = request, read_text, visible assert "Script failed" in read_failure[1], read_failure[1] result_cursor = await _wait_for_cell_result_in_history(deployed, session_id, initial_cursor, read_call_id) await _wait_for_active_history_through(deployed, session_id, result_cursor) @@ -907,7 +910,7 @@ async def gateway_roll_keeps_live_central_read_and_turn( assert read_failure is None request = await asyncio.wait_for(model.request(), timeout=45) assert request.native.client_metadata.session_id == session_id - request, read_text, _ = await _follow_code_mode_cell(model, request, "read-live-process", timeout_s=140) + request, read_text, _, _ = await _follow_code_mode_cell(model, request, "read-live-process", timeout_s=140) assert "Script completed" in read_text, read_text assert "finished\\n" in read_text, "the original in-flight read did not return process output: " + read_text assert '"exit_code":0' in read_text, "the original process did not exit successfully: " + read_text @@ -1055,7 +1058,9 @@ async def gateway_crash_settles_live_central_read_and_accepts_next_input( assert killed_native is None assert read_failure is None request = await asyncio.wait_for(model.request(), timeout=45) - request, read_text, visible = await _follow_code_mode_cell(model, request, "read-crash-process", timeout_s=140) + request, read_text, visible, _ = await _follow_code_mode_cell( + model, request, "read-crash-process", timeout_s=140 + ) if f'retained exec-server session "{native_session_id}" was definitively lost' in visible: assert "no mutation was replayed" in visible, visible await _assert_replacement_native_session_started(recovered, session_id, cursor, native_session_id) @@ -1175,10 +1180,13 @@ async def gateway_retry_exhaustion_reports_unknown_and_accepts_next_input( assert "mutation was not replayed" in visible, visible await model.reply(request, ShellCall("probe-budget-session", "cat gateway-budget-marker")) request = await asyncio.wait_for(model.request(), timeout=45) - probe_outputs = [output for output in request.tool_outputs if output.call_id == "probe-budget-session"] - assert probe_outputs, probe_outputs - assert probe_outputs[0].blocks, probe_outputs - probe_output = probe_outputs[0] + # Service readiness does not end the sampler channel's reconnect backoff. Follow the + # same yielded cell to settlement before asserting its native-session resume; never + # send another exec_command while the original probe is still running. + request, _, _, probe_output = await _follow_code_mode_cell( + model, request, "probe-budget-session", timeout_s=140 + ) + probe_outputs = [output for output in request.tool_outputs if output.call_id == probe_output.call_id] probe_text = "\n".join(probe_output.blocks) if "was definitively lost" in probe_text: assert len(probe_outputs) == 2, probe_outputs @@ -1309,7 +1317,7 @@ async def gateway_outage_restarts_exec_server_then_notifies_model( assert read_failure is None executor_pod, executor_pod_uid, old_exec_server = killed_native request = await asyncio.wait_for(model.request(), timeout=45) - request, read_text, visible = await _follow_code_mode_cell( + request, read_text, visible, _ = await _follow_code_mode_cell( model, request, "read-expiring-process", timeout_s=140 ) assert "retained exec-server session was definitively lost" in visible, visible From 6893ea3810ad12d16959bc12c9385f8eb14d42b1 Mon Sep 17 00:00:00 2001 From: agentydragon-agent <260791330+agentydragon-agent@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:06:26 -0800 Subject: [PATCH 3/5] Remove obsolete Gateway connection records and per-Sandbox routes --- agentplane/docs/runner_control.md | 80 +-- agentplane/docs/system_overview.md | 29 +- agentplane/plans/task_dag.md | 8 +- agentplane/runner_gateway/README.md | 67 +- agentplane/runner_gateway/config.go | 19 +- .../internal/gateway/BUILD.bazel | 13 - .../runner_gateway/internal/gateway/SPEC.md | 45 +- .../internal/gateway/connection_metadata.go | 23 - .../internal/gateway/fixture_test.go | 42 +- .../internal/gateway/gateway.go | 23 +- .../internal/gateway/records.go | 385 ----------- .../internal/gateway/records_test.go | 609 ------------------ .../internal/gateway/runner_observer.go | 45 -- .../internal/gateway/runner_observer_test.go | 121 +--- agentplane/runner_gateway/main.go | 6 +- agentplane/runner_gateway/settings.py | 5 +- agentplane/sandbox_controller/README.md | 14 +- agentplane/sandbox_controller/SPEC.md | 15 +- .../internal/controller/BUILD.bazel | 1 - .../internal/controller/apply.go | 6 - .../internal/controller/cache.go | 3 +- .../controller/children_integration_test.go | 17 +- .../internal/controller/control_service.go | 27 - .../internal/controller/controller.go | 6 - .../internal/controller/firecracker.go | 2 +- .../internal/controller/identity_test.go | 31 - agentplane/sandboxes/api/constants.go | 6 +- .../system_tests/driver/central_sampler.py | 2 +- agentplane/system_tests/driver/sandbox.py | 5 +- .../system_tests/scenarios/dispatcher.py | 7 +- .../cdk8s/agentplane/sandbox_controller.py | 3 +- 31 files changed, 176 insertions(+), 1489 deletions(-) delete mode 100644 agentplane/runner_gateway/internal/gateway/connection_metadata.go delete mode 100644 agentplane/runner_gateway/internal/gateway/records.go delete mode 100644 agentplane/runner_gateway/internal/gateway/records_test.go delete mode 100644 agentplane/sandbox_controller/internal/controller/control_service.go diff --git a/agentplane/docs/runner_control.md b/agentplane/docs/runner_control.md index d21253b9a9c..10bdcfa9309 100644 --- a/agentplane/docs/runner_control.md +++ b/agentplane/docs/runner_control.md @@ -34,29 +34,22 @@ grace. Reconnects start a full idle timer, even when the Runtime reports the sam Quiesce attempts remain persisted, and an unobserved published incarnation must still seal before stop or deletion. Bootstrap failure remains terminal for its incarnation. -Every Gateway replica independently watches the current Controller-bound Runtime with no epoch, -validates the workload and boot ID, and can forward calls. Gateway watches cannot bind or advance -the incarnation. Observation loss removes that replica's callable connection and starts a fresh -attempt after one second; it does not revoke another replica's connection. Replacement observers -wait for their predecessor to exit before publishing their local connection. - -Callers use the stable `runner-gateway` Service. During their rollout, old callers still use -`-control`. The connection Lease and EndpointSlice temporarily -select its destination, independently of Runtime call authority. Each replica attempts route -publication after becoming callable; a live competing route claim does not block its own calls. -Route metadata renews on Runtime snapshots and heartbeats. A contender takes over after observing -an unchanged renewal for one Lease duration on its own clock. Orderly shutdown closes local traffic -and releases its exact claim. These routing records carry no lifecycle facts. Quiesce and -suspend-admission RPCs continue through the Gateway proxy. +Every Gateway replica independently opens a direct `WatchState` to the current Runtime with no +connection epoch. The Controller's authenticated `observe_only` watch binds the Runtime to the +Sandbox UID and accepted workload Pod UID before Gateway watches arrive; a Gateway can observe that +binding but cannot select or advance it. Each replica learns the Runtime boot ID from its own +authenticated watch and can serve calls for the current incarnation. The stable `runner-gateway` +Service routes callers to any replica. Observation loss removes that replica's callable connection +and starts a fresh attempt after one second; it does not revoke another replica's connection. +Replacement observers wait for their predecessor to exit before publishing their local connection. Unary Runner calls and typed `Executor` calls carry the Sandbox UID, incarnation UID and Runtime -boot ID. Calls with an absent or zero `agentplane-connection-epoch` authorize against the -Controller-bound identity, independently of Gateway watch lifetime; any authenticated Gateway -replica may make them. A supplied nonzero epoch retains the legacy requirement for the matching, -live Gateway watch. Stale Sandbox, incarnation, boot ID or legacy epoch fails before admission. -Gateway observer loss no longer cancels epochless calls or their execution-session streams; admitted -journal work remains Runtime-owned and continues independently. `ABORTED` remains the explicit -refusal of new work for a workload incarnation. +boot ID. Runtime checks those values against its Controller-bound identity and process boot; stale +Sandbox, incarnation or boot calls fail before admission. The receiver retains legacy nonzero epoch +tolerance for older Gateway binaries during the rollback window, but the selected Gateway call +contract allocates and sends no epoch. Loss of one replica's observation does not revoke another +replica's connection or cancel Runtime-owned admitted work. `ABORTED` remains the explicit refusal +of new work for a workload incarnation. `Executor.OpenSession` holds the Runner-side Codex connection with a server stream; typed unary process and file RPCs use its opaque session ID. JSON-RPC framing, initialization and native notifications stay in the Runner adapter. Both hops use the same sandbox-owned @@ -71,13 +64,11 @@ runner journal; a repeated ID with a different operation or payload fails `ALREA ## Runner proxy -Every Gateway replica serves `agentplane.env.runtime.control.EnvironmentRuntime` -on its public gRPC port. Callers reach any replica through the stable `runner-gateway` Service. -The Sandbox's `-control` Service remains available for old caller processes during rollout. The proxy forwards -unary calls over the runner gRPC connection and returns the runner's status and response. +Every Gateway replica serves `agentplane.env.runtime.control.EnvironmentRuntime` on the shared gRPC +listener and can forward unary calls over its own direct Runtime connection. Callers reach that +listener through the stable `runner-gateway` Service. -- **Sandbox.** The caller names it in the `x-agentplane-sandbox` header, the same name its target - carries; a gRPC server cannot see the dialed `:authority`. A missing or repeated header is +- **Sandbox.** The caller names it in the `x-agentplane-sandbox` header. A missing or repeated header is `INVALID_ARGUMENT`, an unknown Sandbox `NOT_FOUND`. - **Expected identity.** Callers must supply `x-agentplane-sandbox-uid`, `x-agentplane-sandbox-owner-namespace` and `x-agentplane-sandbox-owner-name` exactly once. @@ -88,8 +79,8 @@ unary calls over the runner gRPC connection and returns the runner's status and verified combined Gateway/Dispatcher workload; clients use SessionDispatcher with their existing direct or delegated authority. Only `sandbox_controller_account` may `Quiesce`; unset, nobody may. Executor permissions stay separate. See the [proxy contract](../runner_gateway/internal/gateway/SPEC.md). -- **Holder.** A replica that does not hold the Sandbox's current connection answers `UNAVAILABLE` - before acting, and the caller dials the control Service again. +- **Replica.** Any Gateway replica with a fresh observation of the current Sandbox incarnation can + serve the request. A replica without a matching local Runtime connection answers `UNAVAILABLE`. - **Deadline.** The proxy propagates the caller's remaining gRPC deadline; a call without one waits until the runner answers or the connection ends. @@ -222,25 +213,18 @@ other watch events or a controller restart from bypassing that deadline. Only af flushed may the Sandbox's volume go. A runner that cannot report leaves the Sandbox in sealing for the operator. -## Legacy routing records in Kubernetes +## Runtime routing -The Gateway temporarily publishes `-environment-runtime` Leases and `-control` -EndpointSlices for existing callers; the Sandbox Controller owns the selector-less control Service. -These records select a route only. Every Gateway replica independently observes and calls Runtime, -regardless of who publishes the route. Lifecycle facts come only from the Controller's direct watch. - -Route updates check the exact Sandbox owner, Lease UID, holder and monotonically increasing route -epoch, with resource-version conflict retries. Foreign records prevent publication but do not block -an authenticated current Runtime connection. The route epoch is never sent to Runtime. A publication -error is logged and retried on subsequent snapshots/heartbeats; a superseded claim is reacquired when -eligible. Existing callers can lose reachability while Kubernetes rejects route writes. A stale route -may reach any still-current replica, which can serve independently; a stopped observer refuses with -`UNAVAILABLE`. Closing one observer does not prove admitted Runtime effects stopped. +Gateway replicas connect to the current Runtime using its Pod IP and private listener. The selected +route does not use a per-Sandbox `-control` EndpointSlice or connection Lease to decide +which replica may call. All caller traffic uses the stable `runner-gateway` Service; the Runtime +identity in each call provides stale-Sandbox, stale-workload and stale-process fencing. The +Dispatcher delivery Lease is unrelated and remains part of durable command scheduling. ## Lifecycle operations The Runner Gateway makes Session calls over the runner gRPC connection. App and Notification callers -reach the stable `runner-gateway` Service (`control_target`, a required setting). +reach any replica through the stable `runner-gateway` Service. - `ListSessions` reads retained Session inventory. - `GetStatus` reads a coherent live snapshot of process incarnations, Session assignments and @@ -258,9 +242,10 @@ reach the stable `runner-gateway` Service (`control_target`, a required setting) It waits for preceding admissions; subsequent new-work calls fail `ABORTED`. Final ingress acknowledgement is required for `QUIESCED`. -Any replica with a current independent observation may call it. Callers reach any replica through the stable Service; old caller processes retain their -per-Sandbox route until rollout completes. Runtime connectivity grants no Dispatcher ownership. Transport reconnect never replays calls; durable command retry uses the -original command ID. Process identities in diagnostics are per-spawn UUIDs, not command addresses. +Any Gateway replica with a current observation may call the Runtime. Runtime connectivity is +independent of Dispatcher scheduling ownership. Transport reconnect never replays calls; durable +command retry uses the original command ID. Process identities in diagnostics are per-spawn UUIDs, +not command addresses. Stored native conversations may be dormant while their store's app server is running. A Session becomes ready after its native conversation preparation/reconciliation, without needing a turn. @@ -280,7 +265,8 @@ Public Gateway callers still use their existing workload bearers and per-method Standalone behavior tests can explicitly select insecure transport at both endpoints. It supplies plaintext without peer authentication; it never fabricates a certificate identity. Live Sandbox/Pod/VM -ownership and incarnation validation, Sandbox/incarnation/boot-ID fencing and command admission remain enforced. +ownership and incarnation validation, Sandbox/incarnation/boot-ID fencing and command admission +remain enforced. Every Sandbox incarnation gets a credential; k3s keeps real cert-manager/trust-manager and mTLS in every stack. Bad mTLS credentials never trigger an insecure retry. diff --git a/agentplane/docs/system_overview.md b/agentplane/docs/system_overview.md index 8a07f1e2588..dd11f9cc2e8 100644 --- a/agentplane/docs/system_overview.md +++ b/agentplane/docs/system_overview.md @@ -207,17 +207,16 @@ embeds files through an embedding endpoint and owns its own PostgreSQL database - **Sessions and Threads.** A Session is a runner's raw Event stream; a Thread is its conversation view, folded by the History Service. Runners push Events to the History Service, the only way into retained history. -- **Runner control uses native gRPC.** The Gateway dials each runner's private server using mTLS. - It validates the certificate against the active Sandbox/workload and retains `-control` - routing and `-environment-runtime` Leases solely for legacy route publication during caller - rollout. Current callers use the stable `runner-gateway` Service. - Every replica independently watches and calls the Runtime without a connection epoch. The Sandbox Controller's - direct mTLS `observe_only` watch supplies the accepted workload UID and binds it for the Runtime - process before the first facts snapshot; the Controller keeps those facts in memory per - Sandbox, incarnation and boot. Gateway watches cannot select or replace that binding. The Gateway - publishes only legacy routing identity and renewal metadata to its Lease; it grants no Runtime call authority. History push and ordinary Sandbox egress retain - their sidecar or VM relay paths; History terminates and verifies the Runtime SVID itself - ([runner control](runner_control.md)). +- **Runner control uses native gRPC.** Every Gateway replica independently dials each current + Runtime's private server using mTLS and opens a zero-epoch state watch. Its calls carry the live + Sandbox UID, accepted workload Pod UID and observed Runtime boot ID. The Sandbox Controller's + direct mTLS `observe_only` watch binds the workload UID before the first facts snapshot; the + Controller keeps those facts in memory per Sandbox, incarnation and boot. Gateway watches cannot + select or replace that binding. Callers use the stable `runner-gateway` Service; no Gateway + connection Lease or per-Sandbox `-control` EndpointSlice decides Runtime call ownership. + The Runtime protocol retains epoch compatibility for older Gateway images during the rollback + window ([runner control](runner_control.md)). History push and ordinary Sandbox egress retain + their sidecar or VM relay paths; History terminates and verifies the Runtime SVID itself. - **Telemetry uses Alloy as its collector.** SDK-instrumented services export OTLP/gRPC to Alloy. Browser spans post to the same-origin App relay, which forwards OTLP/HTTP to Alloy. Runner spans use the sidecar's local telemetry route; VM guest spans use the launcher relay's guest gateway route. @@ -241,8 +240,6 @@ openings and durable command admission/delivery alongside the control stream; th runner inventory and diagnostics through the Runner Proxy, which checks the live Sandbox name, UID and owner. The Runner starts and resumes its own harnesses. -Status, open work and design gates: [`plans/`](../plans/README.md). - ## Focused diagrams Per-path detail for the [system overview above](#system-overview), using the same conventions: @@ -457,8 +454,10 @@ flowchart LR Runtime owns the components and one desired lifecycle revision. Native Session admission rechecks that accepted revision and the public caller epoch before durable work. Component observations prove native cleanup; Execution fencing precedes journal sealing, and exact durable History ACKs -precede delivery/journal closure and owner release. The external control Lease and epoch boundaries -remain in Runtime's endpoint. Central sampler remains a separate control-plane process. +precede delivery/journal closure and owner release. The Runtime endpoint still accepts legacy +nonzero connection epochs for Gateway rollback compatibility; zero-epoch calls use the immutable +Controller-bound Sandbox and incarnation identity plus the Runtime boot ID. Central sampler remains +a separate control-plane process. ## Codex execution inside a Sandbox diff --git a/agentplane/plans/task_dag.md b/agentplane/plans/task_dag.md index 3468316957b..3f23c647951 100644 --- a/agentplane/plans/task_dag.md +++ b/agentplane/plans/task_dag.md @@ -2420,9 +2420,11 @@ Dispatcher and typed Runner RPCs. The Sandbox Controller's authenticated passive Runtime workload UID before it publishes facts. The receiver contract permits any authenticated Gateway replica to call with the bound Sandbox/incarnation/boot identity and an absent or zero epoch; legacy nonzero call and watch epochs remain temporarily supported for that same UID. The -[runtime state observation plan](runtime_state_observation.md) records the contract and rollout. The -direct Controller reader migration merged in #16324; #16328 removes Lease fact publication after -the new Controller is running on staging. Gateway connection removal remains pending. +[runtime state observation plan](runtime_state_observation.md) records the contract and rollout. +The direct Controller reader and Lease-fact removal landed in #16324 and #16328; the receiver +binding and independent-watch compatibility landed in #16345. The remaining work is staged: pin and +validate the receiver before the independent-connection Gateway image, route caller defaults only +after that image is pinned, then remove the old route records after all callers have moved. The [service-boundary contract](../docs/service_boundaries.md) and [Runner Gateway README](../runner_gateway/README.md) describe the current proxy. Live `Runner.ListSessions` remains part of the app and Notification diff --git a/agentplane/runner_gateway/README.md b/agentplane/runner_gateway/README.md index e303138510f..5964936dac9 100644 --- a/agentplane/runner_gateway/README.md +++ b/agentplane/runner_gateway/README.md @@ -5,19 +5,27 @@ The Runner Gateway combines durable Session Dispatcher admission and delivery, t resume and Dispatcher automatic wake use the shared lifecycle manager. Automatic wake preserves the effective caller and applies the Sandbox's `wakeOnSend` setting and the configured `wake_callers` grant. EnvironmentService and Dispatcher share the Gateway listener and Dispatcher TokenReview audience. -The Gateway dials each Runner's native gRPC -server on private port `9090`. One TLS gRPC listener serves Dispatcher, EnvironmentService, -Gateway proxy, Executor and health methods, with independent per-method authorization. -Only the Pod-bound Gateway/Dispatcher workload can proxy Runtime Session creation and commands; -`gateway_callers` grants read-only diagnostics. The selected Runtime call contract allows any -authenticated Gateway replica to connect to and call a Runtime; there is no per-Sandbox connection -owner or connection epoch in that contract. The Runtime fences calls by Sandbox UID, workload -incarnation and Runtime boot ID, all bound from a Runtime-trusted source; the shared Gateway SVID -does not authorize a replica to choose a replacement incarnation. The current image retains its Lease and route epoch solely for legacy caller routing. Startup reviews `runner_token_file` with the Gateway -audience and verifies its Pod name, current UID and ServiceAccount against Kubernetes before serving. -The Kubernetes API must answer that TokenReview and Pod read; failure prevents startup. -See the [proxy contract](internal/gateway/SPEC.md). The Sandbox -Controller owns `Sandbox.status`. +The Gateway dials each Runner's native gRPC server on private port `9090`. Every Gateway replica +independently opens a zero-epoch `WatchState` to each current Runtime and may serve calls for it; +there is no Gateway connection Lease, allocated epoch or per-Sandbox `-control` EndpointSlice. +Callers use the stable `runner-gateway` Service. Each Gateway resolves the live Sandbox and +workload, then calls the Runtime with the Sandbox UID, current workload Pod UID and boot ID +observed from that Runtime. The Sandbox Controller's authenticated direct watch binds the workload +UID before facts are published. The Runtime rejects stale Sandbox, incarnation or boot identities; +the shared Gateway SVID cannot select or advance the binding. The Gateway's per-method caller +authorization remains in force, and uncertain mutations are never replayed automatically. + +The Runtime receiver retains its legacy nonzero watch/call epoch tolerance for the rollback window; +the Gateway does not allocate or send those epochs. +Dispatcher delivery Leases are unchanged. + +One TLS gRPC listener serves Dispatcher, EnvironmentService, Gateway proxy, Executor and health +methods, with independent per-method authorization. Only the Pod-bound Gateway/Dispatcher workload +can proxy Runtime Session creation and commands; `gateway_callers` grants read-only diagnostics. +Startup reviews `runner_token_file` with the Gateway audience and verifies its Pod name, current UID +and ServiceAccount against Kubernetes before serving. The Kubernetes API must answer that +TokenReview and Pod read; failure prevents startup. See the [proxy contract](internal/gateway/SPEC.md). +The Sandbox Controller owns `Sandbox.status`. The required `rpc` settings block configures this shared TLS listener. It enables SPIFFE and Kubernetes bearer authentication, with exactly the Dispatcher and Gateway audiences. Dispatcher @@ -26,16 +34,17 @@ the Gateway audience. Dispatcher and EnvironmentService methods also accept a ve SVID. Each method retains its existing caller allowlist, delegation, and effective-caller checks. Calls select one enabled credential: an invalid presented SVID cannot fall back to bearer authentication, and certificate plus authorization or delegation credentials are rejected as -ambiguous. The Gateway's calls through each Sandbox control Service use `gateway_tls` to verify -the Session Dispatcher server SVID and present only their projected Gateway bearer. +ambiguous. The Gateway uses `gateway_tls` to verify the stable Runner Gateway and Dispatcher service +peers and presents its projected Gateway bearer where required. Cluster and k3s Gateways and Runners use certificates from cert-manager and roots distributed by trust-manager. Required `runner_security.mode: mtls` uses a `tls` block naming the Gateway's client `cert_file`, `key_file` and `ca_file`. The Gateway identity is `spiffe://agentplane/runner-gateway`; runner identities bind the Sandbox UID and its current -controller-assigned credential ID. Mutual TLS plus Sandbox UID, workload incarnation and Runner -boot ID fence every direct Runner RPC. The Gateway connection Lease and epoch are temporary rollout -compatibility, not part of the selected Runtime call fence. +controller-assigned credential ID. Mutual TLS plus Sandbox UID, workload incarnation and Runtime +boot ID fence every direct Runner RPC. Each replica opens an independent zero-epoch Runtime watch; +the retained receiver epoch field is compatibility for older Gateway binaries during rollback and +does not participate in the new call fence. Local tests may explicitly select `runner_security: {mode: insecure}`, with the Runner's `control.security` set to the same value. This disables transport encryption and peer authentication; @@ -46,16 +55,13 @@ Controller still provisions certificates, so cluster and k3s stacks retain real ## Replica ownership decision **Selected by Rai, 2026-11-14 02:36 UTC, relayed by Claude.** Callers route through the stable -`runner-gateway` Service, so any authenticated Gateway replica may receive the call and connect to -the current Runtime. Remove the per-Sandbox `-control` EndpointSlice, which currently routes -to the replica holding the connection Lease. There is no singleton Gateway connection owner or -connection epoch in the selected contract. Runtime fences each request by Sandbox UID, workload -incarnation and boot ID, bound from a Runtime-trusted source; the shared Gateway SVID does not let a -replica choose or advance an incarnation. Per-method caller authorization remains, and uncertain -mutations are never replayed automatically. The Dispatcher delivery Lease is separate and unchanged. -All replicas now independently observe and call the Controller-bound Runtime without a wire epoch. -Callers now use the stable Service. The per-Sandbox route remains until all old caller processes -have rolled out. Its Lease/epoch selects legacy routing only, never Runtime call ownership. +`runner-gateway` Service, so any authenticated Gateway replica may receive a call and connect +directly to the current Runtime. No per-Sandbox `-control` EndpointSlice, Gateway +connection Lease or connection-epoch allocation participates in routing. The Controller-bound +Sandbox UID and workload Pod UID plus the Runtime's boot ID fence each call. The shared Gateway SVID +does not let a replica choose or advance an incarnation. Per-method caller authorization remains, +and uncertain mutations are never replayed automatically. The Dispatcher delivery Lease is separate +and unchanged. The old receiver protocol remains available through the Gateway rollback window. The Dispatcher role retains its PostgreSQL queue, command IDs, per-Sandbox delivery Leases and database ownership generations. Those delivery Leases are separate from Gateway-to-Runtime @@ -64,8 +70,7 @@ no History Service client. The selected Controller ownership has its leader-elec Runtime state directly and keep lifecycle facts in memory. Runtime restart, workload replacement and same-name Sandbox replacement invalidate old calls through boot ID, incarnation UID and Sandbox UID. Observation loss makes facts stale; they cannot prove connectivity, quiescence or idleness. A fresh -snapshot restores observation and resets the Controller's idle timer. The legacy connection Lease and EndpointSlice remain solely for existing caller routing. -Loss of one Gateway observer affects its local connection, while other replicas remain callable. +snapshot restores observation and resets the Controller's idle timer. Every Gateway observer has its own callable connection. Every Gateway forwards an execution mutation at most once. If the direct call loses its connection after the Runtime may have acted, its outcome is uncertain and the Gateway does not automatically @@ -89,7 +94,7 @@ conversation and from Runner admission. `environment_launchers` grants Action brokers create-only authority with required caller origin. `CreateEnvironment` names the Identity the Sandbox acts as and binds no PolicySets: a launch creates the Identity and grants it, including the default sets, before calling it. -- `internal/gateway/`: Runner connections, authenticated proxy, Sandbox/Pod validation and connection ownership. +- `internal/gateway/`: direct Runner connections, authenticated proxy and Sandbox/Pod validation. - `internal/sandboxinventory/`: informer-backed Sandbox reads and the uncached Identity reader that supply the Dispatcher contract. - `dispatcher/`: durable admission, watch, delivery, guarded wake and scheduling ownership. - `migrations/`: Goose migrations for the Dispatcher database. diff --git a/agentplane/runner_gateway/config.go b/agentplane/runner_gateway/config.go index 7bd9f730d18..3ebb4737409 100644 --- a/agentplane/runner_gateway/config.go +++ b/agentplane/runner_gateway/config.go @@ -42,15 +42,16 @@ type config struct { RunnerSecurity peertls.Security `mapstructure:"runner_security"` Namespace string `mapstructure:"sandbox_namespace"` // PolicyNamespace holds the Identities a workload SVID may name. - PolicyNamespace string `mapstructure:"policy_namespace"` - DispatcherDatabaseURL string `mapstructure:"-"` - PodName string `mapstructure:"-"` - PodIP string `mapstructure:"-"` - RunnerTokenFile string `mapstructure:"runner_token_file"` - Kubeconfig string `mapstructure:"kubeconfig"` - KubernetesClientQPS float32 `mapstructure:"kubernetes_client_qps"` - RPC listener.Config `mapstructure:"rpc"` - GatewayTLS *peertls.ClientTLS `mapstructure:"gateway_tls"` + PolicyNamespace string `mapstructure:"policy_namespace"` + DispatcherDatabaseURL string `mapstructure:"-"` + PodName string `mapstructure:"-"` + PodIP string `mapstructure:"-"` + RunnerTokenFile string `mapstructure:"runner_token_file"` + Kubeconfig string `mapstructure:"kubeconfig"` + KubernetesClientQPS float32 `mapstructure:"kubernetes_client_qps"` + RPC listener.Config `mapstructure:"rpc"` + GatewayTLS *peertls.ClientTLS `mapstructure:"gateway_tls"` + // CLEANUP(added 2026-11-14): Remove proxy_port after the route-removal image is pinned. ProxyPort int `mapstructure:"proxy_port"` RunnerControlTargetTemplate string `mapstructure:"runner_control_target_template"` DispatcherAudience string `mapstructure:"dispatcher_audience"` diff --git a/agentplane/runner_gateway/internal/gateway/BUILD.bazel b/agentplane/runner_gateway/internal/gateway/BUILD.bazel index 5b00506dcba..1f31dc0f359 100644 --- a/agentplane/runner_gateway/internal/gateway/BUILD.bazel +++ b/agentplane/runner_gateway/internal/gateway/BUILD.bazel @@ -3,13 +3,11 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "gateway", srcs = [ - "connection_metadata.go", "connections.go", "executor.go", "gateway.go", "peer.go", "proxy.go", - "records.go", "runner_observer.go", "sandboxes.go", ], @@ -30,17 +28,11 @@ go_library( "//agentplane/watchhealth", "@com_github_google_uuid//:uuid", "@com_github_sethvargo_go_retry//:go-retry", - "@io_k8s_api//coordination/v1:coordination", "@io_k8s_api//core/v1:core", - "@io_k8s_api//discovery/v1:discovery", "@io_k8s_apimachinery//pkg/api/errors", "@io_k8s_apimachinery//pkg/api/meta", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", - "@io_k8s_apimachinery//pkg/types", - "@io_k8s_client_go//kubernetes/typed/coordination/v1:coordination", - "@io_k8s_client_go//kubernetes/typed/discovery/v1:discovery", "@io_k8s_client_go//tools/cache", - "@io_k8s_client_go//util/retry", "@io_k8s_client_go//util/workqueue", "@io_k8s_sigs_controller_runtime//pkg/client", "@io_kubevirt_api//core/v1:core", @@ -64,7 +56,6 @@ go_test( "methods_test.go", "peer_test.go", "proxy_test.go", - "records_test.go", "runner_observer_test.go", ], embed = [":gateway"], @@ -84,14 +75,10 @@ go_test( "//agentplane/testing/grpcmethods", "//agentplane/testing/kubeapiserver", "//agentplane/watchhealth", - "@io_k8s_api//coordination/v1:coordination", "@io_k8s_api//core/v1:core", - "@io_k8s_api//discovery/v1:discovery", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_client_go//kubernetes/scheme", - "@io_k8s_client_go//kubernetes/typed/coordination/v1:coordination", - "@io_k8s_client_go//kubernetes/typed/discovery/v1:discovery", "@io_k8s_client_go//rest", "@io_k8s_client_go//tools/cache", "@io_k8s_client_go//util/workqueue", diff --git a/agentplane/runner_gateway/internal/gateway/SPEC.md b/agentplane/runner_gateway/internal/gateway/SPEC.md index 3196d0d3d94..e06eb97593c 100644 --- a/agentplane/runner_gateway/internal/gateway/SPEC.md +++ b/agentplane/runner_gateway/internal/gateway/SPEC.md @@ -1,35 +1,36 @@ # Gateway connection and authorization contracts -## Independent Runtime observations +## Runtime observation and independent Gateway connections -Every authenticated Gateway replica may independently observe and call one Runtime. The Sandbox -Controller binds its Sandbox UID and workload Pod UID through its dedicated observe-only watch; -Gateway watches cannot establish or advance that binding. Gateway watches send no connection epoch. -Calls carry the bound Sandbox UID, incarnation UID and boot ID. Runtime rejects stale fences before -admission. Removing one replica's observation cannot revoke another replica's calls. +Every Gateway replica resolves the current Sandbox and workload independently and opens a direct +`WatchState` to the Runtime on private port `9090`. It uses the Sandbox UID and current workload Pod +UID from Kubernetes and requires the response to match both identities and return a nonempty Runtime +boot ID. A VM's workload Pod is its current virt-launcher Pod. A zero or absent watch epoch can only +observe the incarnation already bound by the Sandbox Controller's authenticated direct watch; the +Gateway cannot establish or advance that binding. Each replica can then serve the Runtime connection +locally. Gateway availability does not depend on per-Sandbox Lease ownership, EndpointSlices or an +epoch allocator. -Observation loss removes only that replica's local callable connection. Reconnect validates the -current workload and learns its boot ID again. Replacement observers wait for their canceled -predecessor to exit before installing a new local connection. Admitted Runtime work remains owned -by Runtime; an uncertain mutation is never automatically replayed. +Each forwarded RPC is fenced by the current Sandbox UID, workload Pod UID and Runtime boot ID. +Those values come from the Gateway's live Kubernetes inventory and its authenticated Runtime +observation, never from caller metadata. Runtime rejects a stale Sandbox, workload incarnation or +process boot before admitting work. Loss of one Gateway's observation closes that replica's local +connection; it does not revoke another replica's connection or cancel Runtime-owned admitted work. +An uncertain mutation is never replayed after reconnect or failover. -## Legacy route handoff +Callers connect through the stable `runner-gateway` Service. The per-Sandbox `-control` +EndpointSlice and Gateway-owned connection Lease are not part of the route. Dispatcher delivery +Leases continue to coordinate queue delivery independently. -The connection Lease and `-control` EndpointSlice temporarily select a destination for -existing callers. Their epoch is a Kubernetes publication fence only; it is never sent to Runtime -and grants no call authority. Each replica becomes callable before attempting route publication, -so another replica's route claim or a publication error does not block its direct calls. - -Orderly shutdown closes local traffic before releasing its exact route claim, fenced by Sandbox -owner, Lease UID, holder and route epoch with resource-version conflict detection. A delayed release -cannot clear a successor. Contenders measure expiry on their own monotonic clocks. Route publication -is retried on snapshots and heartbeats. The Sandbox Controller consumes lifecycle facts directly; -route records prove neither readiness, idleness nor quiescence. +The generated Runtime protocol still carries `connection_epoch` for the old Gateway rollback +window. Legacy nonzero watch and call epochs remain accepted by the receiver for the already-bound +incarnation, but this Gateway sends no allocated epoch. Remove that receiver compatibility and its +RBAC only in the follow-on cleanup after the old Gateway rollback window closes. ## Environment Runtime proxy authorization The public Gateway authenticates Gateway-audience Kubernetes tokens with singular Pod-name and -Pod-UID proof. All calls retain Sandbox UID/owner and current Runtime incarnation fencing. +Pod-UID proof. All calls retain Sandbox UID/owner and current Runtime incarnation and boot fencing. Callers may additionally provide one non-empty `x-agentplane-incarnation-uid` metadata value to require a specific Runtime incarnation. The Gateway resolves and validates the current Sandbox and Pod first, then returns `Aborted` without forwarding if that incarnation differs. A duplicate or diff --git a/agentplane/runner_gateway/internal/gateway/connection_metadata.go b/agentplane/runner_gateway/internal/gateway/connection_metadata.go deleted file mode 100644 index d19f2c84d06..00000000000 --- a/agentplane/runner_gateway/internal/gateway/connection_metadata.go +++ /dev/null @@ -1,23 +0,0 @@ -package gateway - -const ( - environmentRuntimeLeaseSuffix = "-environment-runtime" - environmentRuntimeConnectionLabel = "agentplane.allegedly.works/environment-runtime-connection" - connectionEpochAnnotation = "agentplane.allegedly.works/connection-epoch" - connectionLeaseUIDAnnotation = "agentplane.allegedly.works/connection-lease-uid" - podUIDAnnotation = "agentplane.allegedly.works/pod-uid" - lastConnectedPodUIDAnnotation = "agentplane.allegedly.works/last-connected-pod-uid" - environmentRuntimeBootIDAnnotation = "agentplane.allegedly.works/environment-runtime-boot-id" -) - -// lastConnectedPodUID identifies a verified runner, independently of the current observation claim. -// A boot ID also proves contact on Leases written before the durable annotation was introduced. -func lastConnectedPodUID(annotations map[string]string) string { - if uid := annotations[lastConnectedPodUIDAnnotation]; uid != "" { - return uid - } - if annotations[environmentRuntimeBootIDAnnotation] != "" { - return annotations[podUIDAnnotation] - } - return "" -} diff --git a/agentplane/runner_gateway/internal/gateway/fixture_test.go b/agentplane/runner_gateway/internal/gateway/fixture_test.go index 80adb869fdf..7b5cb003417 100644 --- a/agentplane/runner_gateway/internal/gateway/fixture_test.go +++ b/agentplane/runner_gateway/internal/gateway/fixture_test.go @@ -32,7 +32,6 @@ var ( // api is the API server the binary's tests share; each test's Sandbox is in a namespace of its own. api *kubeapiserver.Server namespaces atomic.Int32 - gateways atomic.Int32 dispatcherAccount = auth.Account{Namespace: "test-dispatchers", Name: "test-dispatcher"} appAccount = auth.Account{Namespace: "test-apps", Name: "test-app"} @@ -169,48 +168,21 @@ func (f *sandboxFixture) newGateway(t *testing.T, executorCallers ...auth.Accoun return f.newGatewayReading(t, f.objects, health, executorCallers...) } -// newGatewayReading is newGateway reading Kubernetes objects through objects, whose health is health. -func (f *sandboxFixture) newGatewayReading( - t *testing.T, objects client.Reader, health *watchhealth.Health, executorCallers ...auth.Account, -) *Gateway { - replica := gateways.Add(1) - return f.newGatewayReplicaReading(t, objects, health, - fmt.Sprintf("test-gateway-%d", replica), fmt.Sprintf("192.0.2.%d", 1+(replica%254)), executorCallers...) -} - -// newGatewayReplicaReading gives the replica an explicit holder identity and EndpointSlice address. -func (f *sandboxFixture) newGatewayReplicaReading( - t *testing.T, objects client.Reader, health *watchhealth.Health, holderName, holderIP string, executorCallers ...auth.Account, -) *Gateway { +func (f *sandboxFixture) newGatewayReading(t *testing.T, objects client.Reader, health *watchhealth.Health, executorCallers ...auth.Account) *Gateway { t.Helper() gateway, err := New(Config{ - Namespace: f.namespace, - RunnerSecurity: peertls.Security{Mode: peertls.MutualTLS, TLS: &f.gatewayTLS}, - CallerAccounts: []auth.Account{appAccount}, - DispatcherAccount: dispatcherAccount, - ExecutorCallers: executorCallers, - SandboxControllerAccount: &controllerAccount, - HolderName: holderName, - HolderIP: holderIP, - ProxyPort: sandboxapi.ControlServicePort, - ControlTiming: ControlTiming{ - LivenessTimeout: 10 * time.Second, - }, - }, f.callerAuth, objects, health, api.Clientset.CoordinationV1(), api.Clientset.DiscoveryV1()) + Namespace: f.namespace, + RunnerSecurity: peertls.Security{Mode: peertls.MutualTLS, TLS: &f.gatewayTLS}, + CallerAccounts: []auth.Account{appAccount}, DispatcherAccount: dispatcherAccount, + ExecutorCallers: executorCallers, SandboxControllerAccount: &controllerAccount, + ControlTiming: ControlTiming{LivenessTimeout: 10 * time.Second}, + }, f.callerAuth, objects, health) if err != nil { t.Fatal(err) } return gateway } -// newGatewayReplica creates one Gateway with the supplied legacy control-route holder identity. -func (f *sandboxFixture) newGatewayReplica(t *testing.T, holderName, holderIP string, executorCallers ...auth.Account) *Gateway { - t.Helper() - health := watchhealth.New() - health.MarkSynced() - return f.newGatewayReplicaReading(t, f.objects, health, holderName, holderIP, executorCallers...) -} - func (f *sandboxFixture) callerContext(token string) context.Context { return metadata.NewOutgoingContext(context.Background(), metadata.Pairs( "authorization", "Bearer "+token, diff --git a/agentplane/runner_gateway/internal/gateway/gateway.go b/agentplane/runner_gateway/internal/gateway/gateway.go index 5f3699ca13e..c6d68445182 100644 --- a/agentplane/runner_gateway/internal/gateway/gateway.go +++ b/agentplane/runner_gateway/internal/gateway/gateway.go @@ -10,8 +10,6 @@ import ( "github.com/agentydragon/ducktape/agentplane/net/peertls" "github.com/agentydragon/ducktape/agentplane/watchhealth" "google.golang.org/grpc" - coordinationclient "k8s.io/client-go/kubernetes/typed/coordination/v1" - discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -30,9 +28,6 @@ type Config struct { // projected token is the only public route to Runtime Session creation and commands. DispatcherAccount auth.Account SandboxControllerAccount *auth.Account - HolderName string - HolderIP string - ProxyPort int32 ControlTiming ControlTiming // ExecutorCallers enables the typed execution API independently from the Runner proxy's // caller allowlist. An empty set leaves the public service unregistered. @@ -52,8 +47,8 @@ func defaultControlTiming(timing ControlTiming) ControlTiming { } func validateControlTiming(timing ControlTiming) error { - if timing.LivenessTimeout <= 0 || timing.LivenessTimeout > time.Duration(leaseDurationSeconds)*time.Second { - return fmt.Errorf("Gateway control timing exceeds the runner liveness or legacy route Lease bound") + if timing.LivenessTimeout <= 0 { + return fmt.Errorf("Gateway control timings must be positive") } return nil } @@ -64,7 +59,6 @@ type Gateway struct { callerAuth *auth.Resolver sandboxes *sandboxResolver connections *connections - records *connectionRecords objects client.Reader // observerQueue holds the names of Sandboxes whose observer RunRunnerObservers must reconcile. observerQueue workqueue.TypedRateLimitingInterface[string] @@ -77,8 +71,6 @@ func New( callerAuth *auth.Resolver, objects client.Reader, objectsHealth *watchhealth.Health, - coordination coordinationclient.CoordinationV1Interface, - discovery discoveryclient.DiscoveryV1Interface, ) (*Gateway, error) { if err := config.RunnerSecurity.Validate(); err != nil { return nil, fmt.Errorf("runner security: %w", err) @@ -89,9 +81,6 @@ func New( if config.DispatcherAccount.Namespace == "" || config.DispatcherAccount.Name == "" { return nil, fmt.Errorf("Gateway Dispatcher account is required") } - if config.HolderName == "" || config.HolderIP == "" || config.ProxyPort < 1 { - return nil, fmt.Errorf("Gateway holder Pod name, IP and proxy port are required") - } config.ControlTiming = defaultControlTiming(config.ControlTiming) if err := validateControlTiming(config.ControlTiming); err != nil { return nil, err @@ -100,12 +89,6 @@ func New( if err != nil { return nil, err } - records, err := newConnectionRecords(coordination, discovery, config.Namespace, connectionHolder{ - name: config.HolderName, ip: config.HolderIP, port: config.ProxyPort, - }) - if err != nil { - return nil, err - } for _, account := range config.CallerAccounts { if account.Namespace == "" || account.Name == "" { return nil, fmt.Errorf("Gateway caller account namespace and name are required") @@ -125,7 +108,7 @@ func New( } return &Gateway{ config: config, callerAuth: callerAuth, objects: objects, - sandboxes: sandboxes, connections: newConnections(), records: records, + sandboxes: sandboxes, connections: newConnections(), observerQueue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]()), }, nil } diff --git a/agentplane/runner_gateway/internal/gateway/records.go b/agentplane/runner_gateway/internal/gateway/records.go deleted file mode 100644 index 0cea82c5d72..00000000000 --- a/agentplane/runner_gateway/internal/gateway/records.go +++ /dev/null @@ -1,385 +0,0 @@ -package gateway - -import ( - "context" - "fmt" - "math" - "strconv" - "sync" - "time" - - sandboxapi "github.com/agentydragon/ducktape/agentplane/sandboxes/api" - "github.com/agentydragon/ducktape/agentplane/sandboxes/sandboxreader" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - coordinationclient "k8s.io/client-go/kubernetes/typed/coordination/v1" - discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1" - "k8s.io/client-go/util/retry" -) - -const ( - endpointSliceManager = "agentplane-runner-gateway" - leaseDurationSeconds = int32(30) -) - -type connectionHolder struct { - name string - ip string - port int32 -} - -// CLEANUP(added 2026-11-14): Remove route records after this any-replica image is -// pinned and callers switch from -control to the Gateway Service. -// These records provide legacy routing only, never Runtime call authority. -type connectionRecords struct { - coordination coordinationclient.CoordinationV1Interface - discovery discoveryclient.DiscoveryV1Interface - namespace string - holder connectionHolder - now func() time.Time - observed leaseObservations -} - -// leaseObservations decides when another replica's connection Lease expires as client-go leaderelection -// does: by how long this replica has seen the same renewal on its own monotonic clock, never by comparing -// the writer's RenewTime with the local clock, so a skewed writer clock neither expires a live claim early -// nor keeps a dead one forever. A replica that has not seen a renewal before waits a full duration. -type leaseObservations struct { - mu sync.Mutex - seen map[string]leaseObservation -} - -type leaseObservation struct { - renewal leaseRenewal - firstSeen time.Time -} - -// leaseRenewal identifies one renewal; any change shows the holder is alive. -type leaseRenewal struct { - uid types.UID - holder string - transitions int32 - renewMicros int64 -} - -// held reports whether lease's holder still holds it at now: whether this replica first saw its current -// renewal less than its duration ago. The caller has checked that holder, RenewTime and duration are set. -func (o *leaseObservations) held(lease *coordinationv1.Lease, now time.Time) bool { - renewal := leaseRenewal{ - uid: lease.UID, holder: *lease.Spec.HolderIdentity, renewMicros: lease.Spec.RenewTime.UnixMicro(), - } - if lease.Spec.LeaseTransitions != nil { - renewal.transitions = *lease.Spec.LeaseTransitions - } - o.mu.Lock() - defer o.mu.Unlock() - observation, ok := o.seen[lease.Name] - if !ok || observation.renewal != renewal { - observation = leaseObservation{renewal: renewal, firstSeen: now} - if o.seen == nil { - o.seen = map[string]leaseObservation{} - } - o.seen[lease.Name] = observation - } - return now.Sub(observation.firstSeen) < time.Duration(*lease.Spec.LeaseDurationSeconds)*time.Second -} - -func (o *leaseObservations) forget(name string) { - o.mu.Lock() - defer o.mu.Unlock() - delete(o.seen, name) -} - -func newConnectionRecords( - coordination coordinationclient.CoordinationV1Interface, - discovery discoveryclient.DiscoveryV1Interface, - namespace string, - holder connectionHolder, -) (*connectionRecords, error) { - if coordination == nil || discovery == nil || namespace == "" || holder.name == "" || holder.ip == "" || holder.port < 1 { - return nil, fmt.Errorf("connection-record clients, namespace and holder are required") - } - return &connectionRecords{coordination: coordination, discovery: discovery, namespace: namespace, holder: holder, now: time.Now}, nil -} - -func (r *connectionRecords) claim(ctx context.Context, sandbox *sandboxreader.Identity, podUID string) (uint64, types.UID, error) { - leaseName := sandbox.Name + environmentRuntimeLeaseSuffix - var epoch uint64 - var leaseUID types.UID - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - current, err := r.coordination.Leases(r.namespace).Get(ctx, leaseName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - current = nil - } else if err != nil { - return err - } - routeEpoch, err := r.controlRouteEpoch(ctx, sandbox) - if err != nil { - return err - } - next := routeEpoch + 1 - if current != nil { - if !ownsSandbox(current.OwnerReferences, sandbox) || !current.DeletionTimestamp.IsZero() { - return fmt.Errorf("refusing to adopt Lease %s: foreign or being deleted", leaseName) - } - if current.Spec.HolderIdentity != nil && *current.Spec.HolderIdentity != r.holder.name && - current.Spec.RenewTime != nil && current.Spec.LeaseDurationSeconds != nil && r.observed.held(current, r.now()) { - return errRunnerOwned - } - if current.Spec.LeaseTransitions != nil { - if *current.Spec.LeaseTransitions < 0 { - return fmt.Errorf("connection Lease %s has invalid transition epoch %d", leaseName, *current.Spec.LeaseTransitions) - } - if uint64(*current.Spec.LeaseTransitions) >= next { - next = uint64(*current.Spec.LeaseTransitions) + 1 - } - } - } - if next == 0 || next > math.MaxInt32 { - return fmt.Errorf("connection epoch exhausted") - } - lease := &coordinationv1.Lease{ - TypeMeta: metav1.TypeMeta{APIVersion: "coordination.k8s.io/v1", Kind: "Lease"}, - ObjectMeta: metav1.ObjectMeta{ - Name: leaseName, Namespace: r.namespace, - Labels: map[string]string{environmentRuntimeConnectionLabel: "true"}, - Annotations: map[string]string{podUIDAnnotation: podUID}, - OwnerReferences: []metav1.OwnerReference{sandboxOwnerReference(sandbox)}, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: stringPointer(r.holder.name), - LeaseDurationSeconds: int32Pointer(leaseDurationSeconds), - LeaseTransitions: int32Pointer(int32(next)), - RenewTime: &metav1.MicroTime{Time: r.now().UTC()}, - }, - } - if current == nil { - lease, err = r.coordination.Leases(r.namespace).Create(ctx, lease, metav1.CreateOptions{}) - } else { - lease.ResourceVersion = current.ResourceVersion - lease.UID = current.UID - if uid := lastConnectedPodUID(current.Annotations); uid != "" { - lease.Annotations[lastConnectedPodUIDAnnotation] = uid - } - lease, err = r.coordination.Leases(r.namespace).Update(ctx, lease, metav1.UpdateOptions{}) - } - if err != nil { - return err - } - if lease.UID == "" { - return fmt.Errorf("connection Lease %s has no UID", leaseName) - } - leaseUID = lease.UID - epoch = next - return nil - }) - if err != nil { - return 0, "", fmt.Errorf("accept runner connection: %w", err) - } - r.observed.forget(leaseName) - if err := r.pointControlEndpoints(ctx, sandbox, leaseUID, epoch); err != nil { - // The Lease was claimed even if publication failed. Return its identity so - // the observer can release that exact claim after closing its connection. - return epoch, leaseUID, fmt.Errorf("point Sandbox control Service at connection holder: %w", err) - } - return epoch, leaseUID, nil -} - -// release gives up only this exact claim after the observer's local traffic and direct Runner -// connection have stopped. A resource-version update and a fresh holder/UID/epoch check on every -// conflict prevent a delayed release from clearing a successor's claim. -func (r *connectionRecords) release(ctx context.Context, sandbox *sandboxreader.Identity, leaseUID types.UID, epoch uint64) error { - name := sandbox.Name + environmentRuntimeLeaseSuffix - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - lease, err := r.coordination.Leases(r.namespace).Get(ctx, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return errSupersededConnection - } - if err != nil { - return err - } - if !ownsSandbox(lease.OwnerReferences, sandbox) || !lease.DeletionTimestamp.IsZero() || lease.UID != leaseUID || - lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != r.holder.name || - lease.Spec.LeaseTransitions == nil || uint64(*lease.Spec.LeaseTransitions) != epoch { - return errSupersededConnection - } - lease.Spec.HolderIdentity = nil - lease.Spec.RenewTime = nil - delete(lease.Annotations, environmentRuntimeBootIDAnnotation) - _, err = r.coordination.Leases(r.namespace).Update(ctx, lease, metav1.UpdateOptions{}) - return err - }) -} - -func (r *connectionRecords) recordBootID(ctx context.Context, sandbox *sandboxreader.Identity, epoch uint64, bootID string) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - lease, err := r.coordination.Leases(r.namespace).Get(ctx, sandbox.Name+environmentRuntimeLeaseSuffix, metav1.GetOptions{}) - if err != nil { - return err - } - if !ownsSandbox(lease.OwnerReferences, sandbox) || lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != r.holder.name || - lease.Spec.LeaseTransitions == nil || uint64(*lease.Spec.LeaseTransitions) != epoch { - return errSupersededConnection - } - if lease.Annotations == nil { - lease.Annotations = make(map[string]string) - } - lease.Annotations[environmentRuntimeBootIDAnnotation] = bootID - lease.Annotations[lastConnectedPodUIDAnnotation] = lease.Annotations[podUIDAnnotation] - _, err = r.coordination.Leases(r.namespace).Update(ctx, lease, metav1.UpdateOptions{}) - return err - }) -} - -func (r *connectionRecords) renew( - ctx context.Context, - sandbox *sandboxreader.Identity, - epoch uint64, - seenAt time.Time, -) error { - name := sandbox.Name + environmentRuntimeLeaseSuffix - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - lease, err := r.coordination.Leases(r.namespace).Get(ctx, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return errSupersededConnection - } - if err != nil { - return err - } - if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != r.holder.name || - lease.Spec.LeaseTransitions == nil || uint64(*lease.Spec.LeaseTransitions) != epoch || - !ownsSandbox(lease.OwnerReferences, sandbox) { - return errSupersededConnection - } - lease.Spec.RenewTime = &metav1.MicroTime{Time: seenAt.UTC()} - _, err = r.coordination.Leases(r.namespace).Update(ctx, lease, metav1.UpdateOptions{}) - return err - }) - if err != nil { - return fmt.Errorf("renew runner connection Lease: %w", err) - } - return nil -} - -func (r *connectionRecords) controlRouteEpoch(ctx context.Context, sandbox *sandboxreader.Identity) (uint64, error) { - name := sandbox.Name + sandboxapi.ControlServiceSuffix - current, err := r.discovery.EndpointSlices(r.namespace).Get(ctx, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return 0, nil - } - if err != nil { - return 0, err - } - if !ownsSandbox(current.OwnerReferences, sandbox) || !current.DeletionTimestamp.IsZero() { - return 0, fmt.Errorf("refusing to adopt EndpointSlice %s: foreign or being deleted", name) - } - routeEpoch, err := parseControlRouteEpoch(current) - if err != nil { - return 0, fmt.Errorf("refusing to adopt EndpointSlice %s with invalid connection epoch: %w", name, err) - } - if routeEpoch >= math.MaxInt32 { - return 0, fmt.Errorf("connection epoch exhausted") - } - return routeEpoch, nil -} - -func (r *connectionRecords) pointControlEndpoints(ctx context.Context, sandbox *sandboxreader.Identity, leaseUID types.UID, epoch uint64) error { - name := sandbox.Name + sandboxapi.ControlServiceSuffix - leaseName := sandbox.Name + environmentRuntimeLeaseSuffix - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - lease, err := r.coordination.Leases(r.namespace).Get(ctx, leaseName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return errSupersededConnection - } - if err != nil { - return err - } - if !ownsSandbox(lease.OwnerReferences, sandbox) || !lease.DeletionTimestamp.IsZero() || lease.UID != leaseUID || - lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != r.holder.name || - lease.Spec.LeaseTransitions == nil || uint64(*lease.Spec.LeaseTransitions) != epoch { - return errSupersededConnection - } - current, err := r.discovery.EndpointSlices(r.namespace).Get(ctx, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - current = nil - } else if err != nil { - return err - } - if current != nil { - if !ownsSandbox(current.OwnerReferences, sandbox) || !current.DeletionTimestamp.IsZero() { - return fmt.Errorf("refusing to adopt EndpointSlice %s: foreign or being deleted", name) - } - currentEpoch, err := parseControlRouteEpoch(current) - if err != nil { - return fmt.Errorf("refusing to update EndpointSlice %s with invalid connection epoch: %w", name, err) - } - if currentEpoch >= epoch { - return nil - } - } - port := r.holder.port - ready := true - slice := &discoveryv1.EndpointSlice{ - TypeMeta: metav1.TypeMeta{APIVersion: "discovery.k8s.io/v1", Kind: "EndpointSlice"}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, Namespace: r.namespace, - Labels: map[string]string{ - "kubernetes.io/service-name": name, - "endpointslice.kubernetes.io/managed-by": endpointSliceManager, - }, - Annotations: map[string]string{ - connectionEpochAnnotation: strconv.FormatUint(epoch, 10), - connectionLeaseUIDAnnotation: string(leaseUID), - }, - OwnerReferences: []metav1.OwnerReference{sandboxOwnerReference(sandbox)}, - }, - AddressType: discoveryv1.AddressTypeIPv4, - Endpoints: []discoveryv1.Endpoint{{ - Addresses: []string{r.holder.ip}, - Conditions: discoveryv1.EndpointConditions{Ready: &ready}, - }}, - Ports: []discoveryv1.EndpointPort{{ - Name: stringPointer(sandboxapi.ControlServicePortName), Port: &port, Protocol: protocolPointer(corev1.ProtocolTCP), - }}, - } - if current == nil { - _, err = r.discovery.EndpointSlices(r.namespace).Create(ctx, slice, metav1.CreateOptions{}) - } else { - slice.ResourceVersion = current.ResourceVersion - _, err = r.discovery.EndpointSlices(r.namespace).Update(ctx, slice, metav1.UpdateOptions{}) - } - return err - }) -} - -// An EndpointSlice created before connection epochs were recorded has no prior high-water mark. -// Preserve that legacy baseline, but never overwrite a present malformed value. -func parseControlRouteEpoch(slice *discoveryv1.EndpointSlice) (uint64, error) { - annotation := slice.Annotations[connectionEpochAnnotation] - if annotation == "" { - return 0, nil - } - return strconv.ParseUint(annotation, 10, 64) -} - -func sandboxOwnerReference(sandbox *sandboxreader.Identity) metav1.OwnerReference { - controller, blockOwnerDeletion := false, false - return metav1.OwnerReference{ - APIVersion: sandboxapi.GroupVersion.String(), Kind: sandboxapi.SandboxKind, Name: sandbox.Name, UID: types.UID(sandbox.Uid), - Controller: &controller, BlockOwnerDeletion: &blockOwnerDeletion, - } -} - -func ownsSandbox(references []metav1.OwnerReference, sandbox *sandboxreader.Identity) bool { - return len(references) == 1 && references[0].APIVersion == sandboxapi.GroupVersion.String() && - references[0].Kind == sandboxapi.SandboxKind && references[0].Name == sandbox.Name && - string(references[0].UID) == sandbox.Uid -} - -func stringPointer(value string) *string { return &value } -func int32Pointer(value int32) *int32 { return &value } -func protocolPointer(value corev1.Protocol) *corev1.Protocol { return &value } diff --git a/agentplane/runner_gateway/internal/gateway/records_test.go b/agentplane/runner_gateway/internal/gateway/records_test.go deleted file mode 100644 index f023aef75d4..00000000000 --- a/agentplane/runner_gateway/internal/gateway/records_test.go +++ /dev/null @@ -1,609 +0,0 @@ -package gateway - -import ( - "context" - "errors" - "fmt" - "maps" - "strings" - "testing" - "time" - - sandboxapi "github.com/agentydragon/ducktape/agentplane/sandboxes/api" - "github.com/agentydragon/ducktape/agentplane/sandboxes/sandboxreader" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - coordinationclient "k8s.io/client-go/kubernetes/typed/coordination/v1" - discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1" -) - -const ( - testSandboxUID = "test-sandbox-uid-a" - testPodUID = "test-runner-pod-uid-a" - // testFinalizer holds an object in deletion for as long as a test needs it there. - testFinalizer = "agentplane.allegedly.works/test-keep" -) - -// recordsFixture is a namespace of the test's own on the shared API server, holding one Sandbox's connection -// records. Nothing runs a garbage collector, so the owner references name a Sandbox no one created. -type recordsFixture struct { - t *testing.T - namespace string - identity *sandboxreader.Identity -} - -func newRecordsFixture(t *testing.T) recordsFixture { - t.Helper() - f := recordsFixture{ - t: t, namespace: fmt.Sprintf("test-records-%d", namespaces.Add(1)), - identity: &sandboxreader.Identity{Name: testSandbox, Uid: testSandboxUID}, - } - namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: f.namespace}} - if _, err := api.Clientset.CoreV1().Namespaces().Create(t.Context(), namespace, metav1.CreateOptions{}); err != nil { - t.Fatal(err) - } - return f -} - -func (f recordsFixture) leases() coordinationclient.LeaseInterface { - return api.Clientset.CoordinationV1().Leases(f.namespace) -} - -func (f recordsFixture) endpointSlices() discoveryclient.EndpointSliceInterface { - return api.Clientset.DiscoveryV1().EndpointSlices(f.namespace) -} - -func (f recordsFixture) records(holder connectionHolder) *connectionRecords { - return f.recordsWith(api.Clientset.CoordinationV1(), api.Clientset.DiscoveryV1(), holder) -} - -func (f recordsFixture) recordsWith( - coordination coordinationclient.CoordinationV1Interface, discovery discoveryclient.DiscoveryV1Interface, holder connectionHolder, -) *connectionRecords { - f.t.Helper() - records, err := newConnectionRecords(coordination, discovery, f.namespace, holder) - if err != nil { - f.t.Fatal(err) - } - return records -} - -func (f recordsFixture) leaseName() string { - return f.identity.Name + environmentRuntimeLeaseSuffix -} - -func (f recordsFixture) routeName() string { return f.identity.Name + sandboxapi.ControlServiceSuffix } - -// createLease stores lease, filling in its namespace, name and owner when unset. -func (f recordsFixture) createLease(lease *coordinationv1.Lease) *coordinationv1.Lease { - f.t.Helper() - lease.Namespace = f.namespace - if lease.Name == "" { - lease.Name = f.leaseName() - } - if lease.OwnerReferences == nil { - lease.OwnerReferences = []metav1.OwnerReference{sandboxOwnerReference(f.identity)} - } - created, err := f.leases().Create(f.t.Context(), lease, metav1.CreateOptions{}) - if err != nil { - f.t.Fatal(err) - } - return created -} - -// route is a control route to address, as a Gateway publishes it; its epoch annotations are the caller's. -func (f recordsFixture) route(address string, annotations map[string]string) *discoveryv1.EndpointSlice { - return &discoveryv1.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.routeName(), Namespace: f.namespace, Annotations: annotations, - OwnerReferences: []metav1.OwnerReference{sandboxOwnerReference(f.identity)}, - }, - AddressType: discoveryv1.AddressTypeIPv4, - Endpoints: []discoveryv1.Endpoint{{Addresses: []string{address}}}, - } -} - -func (f recordsFixture) createRoute(route *discoveryv1.EndpointSlice) *discoveryv1.EndpointSlice { - f.t.Helper() - created, err := f.endpointSlices().Create(f.t.Context(), route, metav1.CreateOptions{}) - if err != nil { - f.t.Fatal(err) - } - return created -} - -func (f recordsFixture) readLease() *coordinationv1.Lease { - f.t.Helper() - lease, err := f.leases().Get(f.t.Context(), f.leaseName(), metav1.GetOptions{}) - if err != nil { - f.t.Fatal(err) - } - return lease -} - -func (f recordsFixture) updateLease(lease *coordinationv1.Lease) { - f.t.Helper() - if _, err := f.leases().Update(f.t.Context(), lease, metav1.UpdateOptions{}); err != nil { - f.t.Fatal(err) - } -} - -func (f recordsFixture) readRoute() *discoveryv1.EndpointSlice { - f.t.Helper() - route, err := f.endpointSlices().Get(f.t.Context(), f.routeName(), metav1.GetOptions{}) - if err != nil { - f.t.Fatal(err) - } - return route -} - -// versions are the resource versions of the namespace's Leases and EndpointSlices, by kind and name. -func (f recordsFixture) versions() map[string]string { - f.t.Helper() - versions := map[string]string{} - leases, err := f.leases().List(f.t.Context(), metav1.ListOptions{}) - if err != nil { - f.t.Fatal(err) - } - for _, lease := range leases.Items { - versions["Lease/"+lease.Name] = lease.ResourceVersion - } - slices, err := f.endpointSlices().List(f.t.Context(), metav1.ListOptions{}) - if err != nil { - f.t.Fatal(err) - } - for _, slice := range slices.Items { - versions["EndpointSlice/"+slice.Name] = slice.ResourceVersion - } - return versions -} - -func TestConnectionRecordsRejectForeignObjects(t *testing.T) { - cases := []struct { - name string - mutate func(*metav1.ObjectMeta) - }{ - {"unowned", func(m *metav1.ObjectMeta) { m.OwnerReferences = []metav1.OwnerReference{} }}, - {"previous Sandbox", func(m *metav1.ObjectMeta) { m.OwnerReferences[0].UID = "previous-sandbox-uid" }}, - {"different API", func(m *metav1.ObjectMeta) { m.OwnerReferences[0].APIVersion = "other.example/v1" }}, - {"different kind", func(m *metav1.ObjectMeta) { m.OwnerReferences[0].Kind = "Other" }}, - {"different name", func(m *metav1.ObjectMeta) { m.OwnerReferences[0].Name = "other-sandbox" }}, - {"multiple owners", func(m *metav1.ObjectMeta) { - m.OwnerReferences = append(m.OwnerReferences, sandboxOwnerReference(&sandboxreader.Identity{Name: "other-sandbox", Uid: "other-sandbox-uid"})) - }}, - {"deleting", func(m *metav1.ObjectMeta) { m.Finalizers = []string{testFinalizer} }}, - } - for _, kind := range []string{"Lease", "EndpointSlice"} { - for _, test := range cases { - t.Run(kind+"/"+test.name, func(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - metadata := metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{sandboxOwnerReference(f.identity)}} - test.mutate(&metadata) - var leaseUID types.UID - if kind == "Lease" { - lease := f.createLease(&coordinationv1.Lease{ObjectMeta: metadata}) - if test.name == "deleting" { - if err := f.leases().Delete(ctx, lease.Name, metav1.DeleteOptions{}); err != nil { - t.Fatal(err) - } - } - } else { - leaseUID = f.createLease(&coordinationv1.Lease{ - Spec: coordinationv1.LeaseSpec{HolderIdentity: stringPointer("gateway"), LeaseTransitions: int32Pointer(1)}, - }).UID - route := f.route("192.0.2.1", nil) - route.OwnerReferences, route.Finalizers = metadata.OwnerReferences, metadata.Finalizers - f.createRoute(route) - if test.name == "deleting" { - if err := f.endpointSlices().Delete(ctx, route.Name, metav1.DeleteOptions{}); err != nil { - t.Fatal(err) - } - } - } - before := f.versions() - records := f.records(connectionHolder{name: "gateway", ip: "10.0.0.1", port: sandboxapi.ControlServicePort}) - var err error - if kind == "Lease" { - _, _, err = records.claim(ctx, f.identity, testPodUID) - } else { - err = records.pointControlEndpoints(ctx, f.identity, leaseUID, 1) - } - if err == nil || !strings.Contains(err.Error(), "refusing") { - t.Fatalf("expected ownership refusal, got %v", err) - } - if after := f.versions(); !maps.Equal(before, after) { - t.Fatalf("refusal changed Kubernetes state: %v, then %v", before, after) - } - }) - } - } -} - -// Another replica's claim expires one Lease duration after this replica last saw it renewed, whatever its -// writer's clock put in RenewTime: a clock behind must not expire a live claim early, nor one ahead keep a -// dead claim for the length of the skew. -func TestClaimExpiresByLocallyObservedRenewal(t *testing.T) { - for _, skew := range []time.Duration{-time.Hour, time.Hour} { - t.Run(skew.String(), func(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - start := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) - writerClock := start.Add(skew) - f.createLease(&coordinationv1.Lease{Spec: coordinationv1.LeaseSpec{ - HolderIdentity: stringPointer("test-other-gateway"), LeaseDurationSeconds: int32Pointer(leaseDurationSeconds), - LeaseTransitions: int32Pointer(1), RenewTime: &metav1.MicroTime{Time: writerClock}, - }}) - records := f.records(connectionHolder{name: "test-gateway", ip: "192.0.2.10", port: sandboxapi.ControlServicePort}) - now := start - records.now = func() time.Time { return now } - duration := time.Duration(leaseDurationSeconds) * time.Second - refused := func(at time.Duration) { - t.Helper() - now = start.Add(at) - if _, _, err := records.claim(ctx, f.identity, testPodUID); !errors.Is(err, errRunnerOwned) { - t.Fatalf("claim at %v took a live connection: %v", at, err) - } - } - refused(0) - refused(duration - time.Second) - // The holder renews by its own clock, which the claim never compares with this one. - renewedAt := duration - time.Second - lease := f.readLease() - lease.Spec.RenewTime = &metav1.MicroTime{Time: writerClock.Add(renewedAt)} - if _, err := f.leases().Update(ctx, lease, metav1.UpdateOptions{}); err != nil { - t.Fatal(err) - } - // Expiry counts from when this replica first sees the renewal, not when it was written. - seenAt := renewedAt + time.Second - refused(seenAt) - refused(seenAt + duration - time.Second) - now = start.Add(seenAt + duration) - epoch, _, err := records.claim(ctx, f.identity, testPodUID) - if err != nil { - t.Fatalf("lapsed connection was not taken over: %v", err) - } - if epoch != 2 { - t.Fatalf("takeover epoch = %d, want 2", epoch) - } - }) - } -} - -func TestConnectionReleaseAllowsImmediateTakeover(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - old := f.records(connectionHolder{name: "old-gateway", ip: "192.0.2.10", port: sandboxapi.ControlServicePort}) - next := f.records(connectionHolder{name: "new-gateway", ip: "192.0.2.11", port: sandboxapi.ControlServicePort}) - epoch, leaseUID, err := old.claim(ctx, f.identity, testPodUID) - if err != nil { - t.Fatal(err) - } - if err := old.recordBootID(ctx, f.identity, epoch, "old-boot"); err != nil { - t.Fatal(err) - } - if _, _, err := next.claim(ctx, f.identity, testPodUID); !errors.Is(err, errRunnerOwned) { - t.Fatalf("live claim was not protected: %v", err) - } - if err := old.release(ctx, f.identity, leaseUID, epoch); err != nil { - t.Fatal(err) - } - lease := f.readLease() - if lease.Spec.HolderIdentity != nil || lease.Spec.RenewTime != nil || *lease.Spec.LeaseTransitions != int32(epoch) || - lease.Annotations[environmentRuntimeBootIDAnnotation] != "" || lastConnectedPodUID(lease.Annotations) != testPodUID { - t.Fatalf("release erased history or retained a live claim: %+v", lease) - } - newEpoch, _, err := next.claim(ctx, f.identity, testPodUID) - if err != nil || newEpoch != epoch+1 { - t.Fatalf("released Lease did not allow immediate takeover: epoch=%d, err=%v", newEpoch, err) - } - route := f.readRoute() - if len(route.Endpoints) != 1 || len(route.Endpoints[0].Addresses) != 1 || route.Endpoints[0].Addresses[0] != "192.0.2.11" { - t.Fatalf("takeover did not repoint the control route: %+v", route.Endpoints) - } -} - -func TestConnectionReleaseCannotClearSupersedingClaim(t *testing.T) { - for _, test := range []struct { - name string - // supersede replaces the old Gateway's claim on lease, as a successor does. - supersede func(f recordsFixture, lease *coordinationv1.Lease) - }{ - {"newer epoch on same holder", func(f recordsFixture, lease *coordinationv1.Lease) { - *lease.Spec.LeaseTransitions++ - f.updateLease(lease) - }}, - {"new holder", func(f recordsFixture, lease *coordinationv1.Lease) { - lease.Spec.HolderIdentity = stringPointer("new-gateway") - f.updateLease(lease) - }}, - {"recreated Lease UID", func(f recordsFixture, lease *coordinationv1.Lease) { - if err := f.leases().Delete(f.t.Context(), lease.Name, metav1.DeleteOptions{}); err != nil { - f.t.Fatal(err) - } - f.createLease(&coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{Annotations: lease.Annotations, Labels: lease.Labels}, - Spec: lease.Spec, - }) - }}, - {"recreated Sandbox UID", func(f recordsFixture, lease *coordinationv1.Lease) { - lease.OwnerReferences[0].UID = "replacement-sandbox" - f.updateLease(lease) - }}, - } { - t.Run(test.name, func(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - old := f.records(connectionHolder{name: "old-gateway", ip: "192.0.2.10", port: sandboxapi.ControlServicePort}) - epoch, leaseUID, err := old.claim(ctx, f.identity, testPodUID) - if err != nil { - t.Fatal(err) - } - test.supersede(f, f.readLease()) - if err := old.release(ctx, f.identity, leaseUID, epoch); !errors.Is(err, errSupersededConnection) { - t.Fatalf("stale release result = %v, want superseded", err) - } - if current := f.readLease(); current.Spec.HolderIdentity == nil || current.Spec.RenewTime == nil { - t.Fatalf("stale release cleared successor: %+v", current.Spec) - } - }) - } -} - -// racingCoordination runs beforeUpdate just before each Lease update reaches the API server, so a test can change -// the Lease under an update already in flight. -type racingCoordination struct { - coordinationclient.CoordinationV1Interface - beforeUpdate func(*coordinationv1.Lease) -} - -func (c racingCoordination) Leases(namespace string) coordinationclient.LeaseInterface { - return racingLeases{LeaseInterface: c.CoordinationV1Interface.Leases(namespace), beforeUpdate: c.beforeUpdate} -} - -type racingLeases struct { - coordinationclient.LeaseInterface - beforeUpdate func(*coordinationv1.Lease) -} - -func (l racingLeases) Update(ctx context.Context, lease *coordinationv1.Lease, options metav1.UpdateOptions) (*coordinationv1.Lease, error) { - l.beforeUpdate(lease) - return l.LeaseInterface.Update(ctx, lease, options) -} - -func TestConnectionReleaseRechecksClaimAfterUpdateConflict(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - holder := connectionHolder{name: "old-gateway", ip: "192.0.2.10", port: sandboxapi.ControlServicePort} - epoch, leaseUID, err := f.records(holder).claim(ctx, f.identity, testPodUID) - if err != nil { - t.Fatal(err) - } - // A successor claims the Lease after the release read it and before its update lands, so the update conflicts. - intercepted := false - old := f.recordsWith(racingCoordination{ - CoordinationV1Interface: api.Clientset.CoordinationV1(), - beforeUpdate: func(*coordinationv1.Lease) { - if intercepted { - return - } - intercepted = true - successor := f.readLease() - successor.Spec.HolderIdentity = stringPointer("new-gateway") - *successor.Spec.LeaseTransitions++ - f.updateLease(successor) - }, - }, api.Clientset.DiscoveryV1(), holder) - if err := old.release(ctx, f.identity, leaseUID, epoch); !errors.Is(err, errSupersededConnection) || !intercepted { - t.Fatalf("racing release = %v, intercepted=%t", err, intercepted) - } - current := f.readLease() - if current.Spec.HolderIdentity == nil || *current.Spec.HolderIdentity != "new-gateway" || - *current.Spec.LeaseTransitions != int32(epoch+1) || current.Spec.RenewTime == nil { - t.Fatalf("stale release cleared the racing successor: %+v", current.Spec) - } -} - -func TestConnectionRenewPreservesOwnershipAndVerifiedIdentity(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - records := f.records(connectionHolder{name: "test-gateway", ip: "192.0.2.10", port: sandboxapi.ControlServicePort}) - epoch, _, err := records.claim(ctx, f.identity, testPodUID) - if err != nil { - t.Fatal(err) - } - if err := records.recordBootID(ctx, f.identity, epoch, "verified-boot"); err != nil { - t.Fatal(err) - } - before := f.readLease() - seenAt := before.Spec.RenewTime.Add(time.Second) - if err := records.renew(ctx, f.identity, epoch, seenAt); err != nil { - t.Fatal(err) - } - after := f.readLease() - if after.UID != before.UID || *after.Spec.HolderIdentity != *before.Spec.HolderIdentity || - *after.Spec.LeaseTransitions != *before.Spec.LeaseTransitions || !maps.Equal(after.Annotations, before.Annotations) || - !after.Spec.RenewTime.Time.Equal(seenAt) { - t.Fatalf("renewal changed ownership or identity: before=%+v after=%+v", before, after) - } -} - -func TestObservationClaimsPreserveVerifiedContact(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - records := f.records(connectionHolder{name: "gateway", ip: "192.0.2.10", port: sandboxapi.ControlServicePort}) - epoch, _, err := records.claim(ctx, f.identity, testPodUID) - if err != nil { - t.Fatal(err) - } - if lastConnectedPodUID(f.readLease().Annotations) != "" { - t.Fatal("claim counted as contact") - } - if err := records.recordBootID(ctx, f.identity, epoch, "boot"); err != nil { - t.Fatal(err) - } - for _, uid := range []string{testPodUID, "replacement-pod"} { - epoch, _, err = records.claim(ctx, f.identity, uid) - if err != nil { - t.Fatal(err) - } - lease := f.readLease() - if lease.Annotations[environmentRuntimeBootIDAnnotation] != "" { - t.Fatal("claim retained current boot ID") - } - if lastConnectedPodUID(lease.Annotations) != testPodUID { - t.Fatal("claim erased verified contact") - } - } - if err := records.recordBootID(ctx, f.identity, epoch, "replacement-boot"); err != nil { - t.Fatal(err) - } - if lastConnectedPodUID(f.readLease().Annotations) != "replacement-pod" { - t.Fatal("verified replacement did not update contact") - } -} - -func TestControlRouteRecoversAfterLeaseRecreation(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - oldLease := f.createLease(&coordinationv1.Lease{Spec: coordinationv1.LeaseSpec{ - HolderIdentity: stringPointer("test-old-gateway"), LeaseTransitions: int32Pointer(9), - }}) - f.createRoute(f.route("192.0.2.40", map[string]string{ - connectionEpochAnnotation: "9", - connectionLeaseUIDAnnotation: string(oldLease.UID), - })) - if err := f.leases().Delete(ctx, oldLease.Name, metav1.DeleteOptions{}); err != nil { - t.Fatal(err) - } - current := f.records(connectionHolder{name: "test-new-gateway", ip: "192.0.2.41", port: sandboxapi.ControlServicePort}) - epoch, _, err := current.claim(ctx, f.identity, testPodUID) - if err != nil { - t.Fatal(err) - } - if epoch != 10 { - t.Fatalf("recreated Lease epoch = %d, want 10 above the retained route", epoch) - } - lease := f.readLease() - if lease.UID == oldLease.UID || *lease.Spec.LeaseTransitions != 10 { - t.Fatalf("recreated Lease = UID %q, transitions %d", lease.UID, *lease.Spec.LeaseTransitions) - } - if err := current.pointControlEndpoints(ctx, f.identity, oldLease.UID, 9); !errors.Is(err, errSupersededConnection) { - t.Fatalf("old Lease publication error = %v, want superseded", err) - } - updated := f.readRoute() - if updated.Annotations[connectionEpochAnnotation] != "10" || - updated.Annotations[connectionLeaseUIDAnnotation] != string(lease.UID) || - len(updated.Endpoints) != 1 || updated.Endpoints[0].Addresses[0] != "192.0.2.41" { - t.Fatalf("control route after Lease recreation = %+v", updated) - } -} - -func TestControlRouteEpochUsesLegacyBaselineAndRejectsInvalidValues(t *testing.T) { - cases := []struct { - name string - annotation string - wantEpoch uint64 - wantError bool - }{ - {name: "legacy absent annotation", wantEpoch: 0}, - {name: "malformed", annotation: "not-an-epoch", wantError: true}, - {name: "overflow", annotation: "2147483648", wantError: true}, - } - for _, test := range cases { - t.Run(test.name, func(t *testing.T) { - f := newRecordsFixture(t) - var annotations map[string]string - if test.annotation != "" { - annotations = map[string]string{connectionEpochAnnotation: test.annotation} - } - f.createRoute(f.route("192.0.2.60", annotations)) - records := f.records(connectionHolder{name: "gateway", ip: "192.0.2.60", port: sandboxapi.ControlServicePort}) - epoch, err := records.controlRouteEpoch(t.Context(), f.identity) - if test.wantError { - if err == nil { - t.Fatal("expected invalid epoch to fail closed") - } - return - } - if err != nil || epoch != test.wantEpoch { - t.Fatalf("controlRouteEpoch() = (%d, %v), want (%d, nil)", epoch, err, test.wantEpoch) - } - }) - } -} - -// racingDiscovery runs beforeUpdate just before each control-route update reaches the API server, so a test can -// change the route under a publication already in flight. -type racingDiscovery struct { - discoveryclient.DiscoveryV1Interface - beforeUpdate func(*discoveryv1.EndpointSlice) -} - -func (d racingDiscovery) EndpointSlices(namespace string) discoveryclient.EndpointSliceInterface { - return racingEndpointSlices{EndpointSliceInterface: d.DiscoveryV1Interface.EndpointSlices(namespace), beforeUpdate: d.beforeUpdate} -} - -type racingEndpointSlices struct { - discoveryclient.EndpointSliceInterface - beforeUpdate func(*discoveryv1.EndpointSlice) -} - -func (s racingEndpointSlices) Update(ctx context.Context, slice *discoveryv1.EndpointSlice, options metav1.UpdateOptions) (*discoveryv1.EndpointSlice, error) { - s.beforeUpdate(slice) - return s.EndpointSliceInterface.Update(ctx, slice, options) -} - -func TestDelayedRoutePublicationCannotRegressRecreatedLeaseRoute(t *testing.T) { - ctx := t.Context() - f := newRecordsFixture(t) - oldLease := f.createLease(&coordinationv1.Lease{Spec: coordinationv1.LeaseSpec{ - HolderIdentity: stringPointer("test-old-gateway"), LeaseTransitions: int32Pointer(2), - }}) - f.createRoute(f.route("192.0.2.50", map[string]string{ - connectionEpochAnnotation: "1", - connectionLeaseUIDAnnotation: string(oldLease.UID), - })) - // While the old Gateway's publication is in flight, a new Gateway recreates the Lease and publishes its route. - var newLease *coordinationv1.Lease - transitioned := false - oldRecords := f.recordsWith(api.Clientset.CoordinationV1(), racingDiscovery{ - DiscoveryV1Interface: api.Clientset.DiscoveryV1(), - beforeUpdate: func(slice *discoveryv1.EndpointSlice) { - if transitioned || slice.Endpoints[0].Addresses[0] != "192.0.2.50" { - return - } - transitioned = true - if err := f.leases().Delete(ctx, oldLease.Name, metav1.DeleteOptions{}); err != nil { - t.Fatal(err) - } - newLease = f.createLease(&coordinationv1.Lease{Spec: coordinationv1.LeaseSpec{ - HolderIdentity: stringPointer("test-new-gateway"), LeaseTransitions: int32Pointer(2), - }}) - route := f.readRoute() - route.Annotations = map[string]string{ - connectionEpochAnnotation: "2", - connectionLeaseUIDAnnotation: string(newLease.UID), - } - route.Endpoints[0].Addresses[0] = "192.0.2.51" - if _, err := f.endpointSlices().Update(ctx, route, metav1.UpdateOptions{}); err != nil { - t.Fatal(err) - } - }, - }, connectionHolder{name: "test-old-gateway", ip: "192.0.2.50", port: sandboxapi.ControlServicePort}) - if err := oldRecords.pointControlEndpoints(ctx, f.identity, oldLease.UID, 2); !errors.Is(err, errSupersededConnection) { - t.Fatalf("delayed old publication error = %v, want superseded", err) - } - if !transitioned { - t.Fatal("the old publication never reached its update") - } - updated := f.readRoute() - if updated.Annotations[connectionLeaseUIDAnnotation] != string(newLease.UID) || - updated.Annotations[connectionEpochAnnotation] != "2" || - updated.Endpoints[0].Addresses[0] != "192.0.2.51" { - t.Fatalf("delayed old publication regressed control route: %+v", updated) - } -} diff --git a/agentplane/runner_gateway/internal/gateway/runner_observer.go b/agentplane/runner_gateway/internal/gateway/runner_observer.go index 3960aa345d6..1a121afd099 100644 --- a/agentplane/runner_gateway/internal/gateway/runner_observer.go +++ b/agentplane/runner_gateway/internal/gateway/runner_observer.go @@ -2,7 +2,6 @@ package gateway import ( "context" - "errors" "fmt" "log/slog" "sync" @@ -20,14 +19,10 @@ import ( "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" toolscache "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" ) -// These records preserve legacy caller routing only; they grant no Runtime call authority. -var errRunnerOwned = status.Error(codes.Unavailable, "another Gateway publishes this Sandbox control route") - type fencedRunnerConn struct { conn *grpc.ClientConn identity metadata.MD @@ -208,21 +203,12 @@ func (g *Gateway) observeRunner(ctx context.Context, destination runnerDestinati return err } var connection *runnerConnection - var routeEpoch uint64 - var routeLeaseUID types.UID defer func() { if connection != nil { g.connections.remove(connection, status.Error(codes.Unavailable, "runner observation ended")) } cancel() _ = conn.Close() - if routeLeaseUID != "" { - releaseCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) - defer stop() - if err := g.records.release(releaseCtx, destination.identity, routeLeaseUID, routeEpoch); err != nil && !errors.Is(err, errSupersededConnection) { - slog.Warn("legacy control route release failed", "sandbox", destination.identity.Name, "error", err) - } - } }() watch, err := environmentruntimepb.NewEnvironmentRuntimeClient(conn).WatchState(ctx, &environmentruntimepb.WatchStateRequest{ SandboxUid: destination.identity.Uid, IncarnationUid: destination.podUID, @@ -291,11 +277,6 @@ func (g *Gateway) observeRunner(ctx context.Context, destination runnerDestinati attribute.String("runtime.boot_id", response.EnvironmentRuntimeBootId), )) } - if err == nil { - if routeErr := g.maintainLegacyRoute(validateCtx, destination, &routeLeaseUID, &routeEpoch, response.EnvironmentRuntimeBootId); routeErr != nil { - slog.Warn("legacy control route publication failed", "sandbox", destination.identity.Name, "error", routeErr) - } - } release() if err != nil { return err @@ -310,29 +291,3 @@ func (g *Gateway) observeRunner(ctx context.Context, destination runnerDestinati } } } - -// maintainLegacyRoute keeps old per-Sandbox callers usable until the Service-routing -// activation follows this image pin. A route claim never gates this replica's calls. -func (g *Gateway) maintainLegacyRoute(ctx context.Context, destination runnerDestination, leaseUID *types.UID, epoch *uint64, bootID string) error { - if *leaseUID == "" { - next, uid, err := g.records.claim(ctx, destination.identity, destination.podUID) - *epoch, *leaseUID = next, uid - if errors.Is(err, errRunnerOwned) { - return nil - } - if err != nil { - return err - } - } - if err := g.records.renew(ctx, destination.identity, *epoch, time.Now()); err != nil { - if errors.Is(err, errSupersededConnection) { - *leaseUID, *epoch = "", 0 - return nil - } - return err - } - if err := g.records.pointControlEndpoints(ctx, destination.identity, *leaseUID, *epoch); err != nil { - return err - } - return g.records.recordBootID(ctx, destination.identity, *epoch, bootID) -} diff --git a/agentplane/runner_gateway/internal/gateway/runner_observer_test.go b/agentplane/runner_gateway/internal/gateway/runner_observer_test.go index a28f9e2c0d7..e24d3b544a0 100644 --- a/agentplane/runner_gateway/internal/gateway/runner_observer_test.go +++ b/agentplane/runner_gateway/internal/gateway/runner_observer_test.go @@ -50,33 +50,6 @@ type observedRunner struct { quiesceCalls atomic.Int32 } -type statusOnlyRuntime struct { - environmentruntimepb.UnimplementedEnvironmentRuntimeServer - calls atomic.Int32 - changes chan struct{} - changesMu sync.Mutex - setupsRunning atomic.Bool -} - -func (r *statusOnlyRuntime) GetStatus(context.Context, *sessionpb.GetStatusRequest) (*sessionpb.EnvironmentRuntimeStatus, error) { - r.calls.Add(1) - return &sessionpb.EnvironmentRuntimeStatus{}, nil -} - -func (r *statusOnlyRuntime) currentChanges() <-chan struct{} { - r.changesMu.Lock() - defer r.changesMu.Unlock() - return r.changes -} - -func (r *statusOnlyRuntime) notify() { - r.setupsRunning.Store(true) - r.changesMu.Lock() - defer r.changesMu.Unlock() - close(r.changes) - r.changes = make(chan struct{}) -} - func (r *observedRunner) WatchState(request *environmentruntimepb.WatchStateRequest, stream grpc.ServerStreamingServer[environmentruntimepb.WatchStateResponse]) error { if request.GetSandboxUid() != r.sandboxUID || request.GetIncarnationUid() != r.incarnationUID || request.GetConnectionEpoch() != 0 || request.GetObserveOnly() { @@ -140,8 +113,8 @@ func TestRunnerObserverPublishesIdentityAndFencesRPCs(t *testing.T) { func TestAnyGatewayReplicaObservesAndCallsOneRuntime(t *testing.T) { f := newSandboxFixture(t) - gatewayA := f.newGatewayReplica(t, "gateway-a", "192.0.2.10") - gatewayB := f.newGatewayReplica(t, "gateway-b", "192.0.2.11") + gatewayA := f.newGateway(t) + gatewayB := f.newGateway(t) destinationA, backend := startProductionRuntime(t, f, gatewayA) destinationB, err := gatewayB.runnerDestination(t.Context(), f.sandbox) if err != nil { @@ -231,82 +204,14 @@ func TestAnyGatewayReplicaObservesAndCallsOneRuntime(t *testing.T) { } } -func TestRunnerObservationRetriesLegacyRouteAfterAnotherReplicaReleases(t *testing.T) { - f := newSandboxFixture(t) - gatewayA := f.newGatewayReplica(t, "gateway-a", "192.0.2.10") - gatewayB := f.newGatewayReplica(t, "gateway-b", "192.0.2.11") - destinationA, backend := startProductionRuntime(t, f, gatewayA) - destinationB, err := gatewayB.runnerDestination(t.Context(), f.sandbox) - if err != nil { - t.Fatal(err) - } - destinationB.target = destinationA.target - - cancelA, doneA, _ := runObserver(t, gatewayA, destinationA) - awaitConnection(t, gatewayA, string(f.sandbox.UID), "shared-runtime-boot", doneA) - awaitEndpointAddress := func(want string) { - t.Helper() - deadline := time.NewTimer(5 * time.Second) - defer deadline.Stop() - poll := time.NewTicker(time.Millisecond) - defer poll.Stop() - for { - route, err := api.Clientset.DiscoveryV1().EndpointSlices(f.namespace).Get( - t.Context(), f.sandbox.Name+sandboxapi.ControlServiceSuffix, metav1.GetOptions{}, - ) - if err == nil && len(route.Endpoints) == 1 && len(route.Endpoints[0].Addresses) == 1 && route.Endpoints[0].Addresses[0] == want { - return - } - select { - case <-deadline.C: - t.Fatalf("legacy control route did not point at %s", want) - case <-poll.C: - } - } - } - awaitEndpointAddress(gatewayA.config.HolderIP) - readLeaseHolder := func() string { - t.Helper() - lease, err := api.Clientset.CoordinationV1().Leases(f.namespace).Get( - t.Context(), f.sandbox.Name+environmentRuntimeLeaseSuffix, metav1.GetOptions{}, - ) - if err != nil { - t.Fatal(err) - } - if lease.Spec.HolderIdentity == nil { - return "" - } - return *lease.Spec.HolderIdentity - } - if got := readLeaseHolder(); got != gatewayA.config.HolderName { - t.Fatalf("initial legacy route holder = %q, want %q", got, gatewayA.config.HolderName) - } - - cancelB, doneB, _ := runObserver(t, gatewayB, destinationB) - defer cancelB() - awaitConnection(t, gatewayB, string(f.sandbox.UID), "shared-runtime-boot", doneB) - clientB := startProxyClient(t, gatewayB) - if _, err := clientB.GetStatus(f.callerContext(f.appToken), &sessionpb.GetStatusRequest{}); err != nil { - t.Fatalf("Gateway B observation was blocked by Gateway A's legacy route claim: %v", err) - } - awaitEndpointAddress(gatewayA.config.HolderIP) - if got := readLeaseHolder(); got != gatewayA.config.HolderName { - t.Fatalf("Gateway B changed a live legacy route claim: holder = %q, want %q", got, gatewayA.config.HolderName) - } +type statusOnlyRuntime struct { + environmentruntimepb.UnimplementedEnvironmentRuntimeServer + calls atomic.Int32 +} - cancelA() - waitObserver(t, doneA) - backend.notify() - awaitEndpointAddress(gatewayB.config.HolderIP) - if got := readLeaseHolder(); got != gatewayB.config.HolderName { - t.Fatalf("legacy route holder after Gateway A released = %q, want %q", got, gatewayB.config.HolderName) - } - if _, err := clientB.GetStatus(f.callerContext(f.appToken), &sessionpb.GetStatusRequest{}); err != nil { - t.Fatalf("Gateway B stopped serving after taking over the legacy route: %v", err) - } - if got := backend.calls.Load(); got != 2 { - t.Fatalf("shared Runtime received %d Gateway B calls, want 2", got) - } +func (r *statusOnlyRuntime) GetStatus(context.Context, *sessionpb.GetStatusRequest) (*sessionpb.EnvironmentRuntimeStatus, error) { + r.calls.Add(1) + return &sessionpb.EnvironmentRuntimeStatus{}, nil } func startProductionRuntime(t *testing.T, f *sandboxFixture, g *Gateway) (runnerDestination, *statusOnlyRuntime) { @@ -320,17 +225,17 @@ func startProductionRuntime(t *testing.T, f *sandboxFixture, g *Gateway) (runner t.Fatal(err) } t.Cleanup(func() { _ = opened.Close() }) - backend := &statusOnlyRuntime{changes: make(chan struct{})} + backend := &statusOnlyRuntime{} + changes := make(chan struct{}) runtimeServer := runtimecontrol.New(runtimecontrol.Config{ SandboxUID: destination.identity.Uid, BootID: "shared-runtime-boot", ListenAddress: "127.0.0.1:0", Security: opened, IsClosing: func() bool { return false }, AdoptIncarnation: func(context.Context, string) error { return nil }, Facts: func() (*environmentruntimestatepb.EnvironmentRuntimeFacts, error) { return &environmentruntimestatepb.EnvironmentRuntimeFacts{ - Bootstrap: environmentruntimestatepb.EnvironmentRuntimeFacts_BOOTSTRAP_SUCCEEDED, - SetupsRunning: backend.setupsRunning.Load(), + Bootstrap: environmentruntimestatepb.EnvironmentRuntimeFacts_BOOTSTRAP_SUCCEEDED, }, nil }, - Changes: backend.currentChanges, + Changes: func() <-chan struct{} { return changes }, }, backend) listener, server, err := runtimeServer.Listen(&executorpb.UnimplementedExecutorServer{}) if err != nil { diff --git a/agentplane/runner_gateway/main.go b/agentplane/runner_gateway/main.go index 96c09325297..a025cd176b8 100644 --- a/agentplane/runner_gateway/main.go +++ b/agentplane/runner_gateway/main.go @@ -311,8 +311,7 @@ func serve(ctx context.Context, c config, rpcListener net.Listener) error { Namespace: c.Namespace, CallerAccounts: c.GatewayCallers, ExecutorCallers: c.ExecutorCallers, DispatcherAccount: deliveryAccount, SandboxControllerAccount: c.ControllerAccount, - HolderName: c.PodName, HolderIP: c.PodIP, ProxyPort: int32(c.ProxyPort), - }, gatewayRPCAuth, inventory, inventoryHealth, kube.CoordinationV1(), kube.DiscoveryV1()) + }, gatewayRPCAuth, inventory, inventoryHealth) if err != nil { return err } @@ -460,8 +459,7 @@ func serve(ctx context.Context, c config, rpcListener net.Listener) error { closing, release := context.WithTimeout(context.Background(), 5*time.Second) defer release() graceful.Stop(closing, server) - // Observer shutdown closes direct Runner connections before releasing their - // fenced Leases. Keep the process alive for that bounded handoff. + // Finish observer shutdown so this replica closes its direct Runtime streams. select { case <-observersStopped: case <-time.After(10 * time.Second): diff --git a/agentplane/runner_gateway/settings.py b/agentplane/runner_gateway/settings.py index 7a8c1cd17f9..4234c9fc30c 100644 --- a/agentplane/runner_gateway/settings.py +++ b/agentplane/runner_gateway/settings.py @@ -74,7 +74,10 @@ class Settings(BaseModel): description="Verify the Session Dispatcher SVID on this Gateway's named control-Service calls." ) proxy_port: int = Field( - default=8080, ge=1, le=65535, description="Port advertised in each Sandbox control EndpointSlice." + default=8080, + ge=1, + le=65535, + description="Legacy control EndpointSlice port, retained until the route-removal image is pinned.", ) runner_control_target_template: str = Field( default="runner-gateway.{namespace}.svc:8080", diff --git a/agentplane/sandbox_controller/README.md b/agentplane/sandbox_controller/README.md index 78adee7b516..f3ae2dea470 100644 --- a/agentplane/sandbox_controller/README.md +++ b/agentplane/sandbox_controller/README.md @@ -42,7 +42,7 @@ A [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) co check them directly. The pinned upstream `controller-gen` binary and Go SDK run with declared Bazel inputs, so no generated Go file is committed. - [`controller.go`](internal/controller/controller.go): one reconcile per `Sandbox`. Its VM (`Owns`), its ConfigMap, - Service and WorkloadCredential, its Pods, VMI and state PVC (by the Sandbox label), and the + WorkloadCredential, its Pods, VMI and state PVC (by the Sandbox label), and the templates and Identity it names (by a field index) map back to it. - [`vm.go`](internal/controller/vm.go): a VM Sandbox's VirtualMachine (types from `kubevirt.io/api`), its run strategy, and the VMI and virt-launcher Pod its status comes from. @@ -55,9 +55,11 @@ A [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) co the next start; a running incarnation with one is replaced. - [`cmd/main.go`](cmd/main.go): the manager, its cache limited to one namespace. - [`cache.go`](internal/controller/cache.go): the cache holds only objects carrying the Sandbox label (ConfigMaps, - Services and the other kinds it creates or watches), every Identity and WorkloadCredential in the + and the other kinds it creates or watches), every Identity and WorkloadCredential in the namespace and Pod Events. Runtime facts come only from direct in-memory observation. - Connection Lease reads are no longer cached; their RBAC awaits the deployed image pin. + Connection Lease reads are no longer cached. Lease RBAC is retained through the Gateway routing + cutover and removed in a follow-up after the final Gateway image pin; leader election continues + to use its own Lease. RBAC cannot select by label, so the Role still grants `list`/`watch` on those kinds. @@ -126,9 +128,9 @@ on `:8081`, and replicas elect a leader by a Lease in the namespace. ineligible immediately; connectivity keeps a one-minute grace. Reopening a stream restarts its full idle timer, including when `idle-since` is unchanged. Quiesce retry deadlines remain durable; only fresh observed quiescence or an acknowledged quiesce RPC completes a seal. Bootstrap failure - stays terminal for that incarnation. Every Gateway replica can now call independently with Sandbox UID, incarnation and boot ID; - the current legacy route still uses its connection Lease - and control EndpointSlice; its legacy nonzero watch can observe only the Controller-bound UID. + stays terminal for that incarnation. Every Gateway replica independently observes the + Controller-bound Runtime UID with a zero-epoch watch and may call using the current Sandbox UID, + workload UID and Runtime boot ID. - **Gotcha:** a deleted Sandbox whose runner cannot seal stays in `Quiescing`. To give up on the seal, remove the finalizer by hand: the PVC is owned, so it goes with the Sandbox, and the Pod or VM with it. diff --git a/agentplane/sandbox_controller/SPEC.md b/agentplane/sandbox_controller/SPEC.md index 1f64103370a..ab6c6c8eb7a 100644 --- a/agentplane/sandbox_controller/SPEC.md +++ b/agentplane/sandbox_controller/SPEC.md @@ -248,13 +248,14 @@ Gateway token; token contents remain in the rotating read-only projected volume. cold start is not implemented yet. - A deleted Sandbox that must run once more to seal waits for a live Identity. -## Control Service +## Runner Gateway calls -Every Sandbox has a headless, selector-less `-control` Service on the Runner Gateway's -public proxy port, 8080. Callers use their workload bearers to reach any replica through -the stable Runner Gateway Service. The per-Sandbox Service remains available for old caller processes -during rollout. The Gateway connects directly to the current workload's mTLS runner listener on port 9090. The controller repairs the per-Sandbox Service's managed fields; the Gateway owns its EndpointSlice. -A foreign same-name Service blocks new workload starts. +The controller uses `runner_gateway_target` to call the stable `runner-gateway` Service while sealing +a Sandbox with `Runner.Quiesce`. It supplies the Sandbox identity and incarnation UID on each call; +the receiving Gateway replica routes the request to its local runner connection. Sandbox Controller +creates no per-Sandbox control Service. Every Gateway replica maintains its own direct Runtime +watch; the Runtime's Sandbox UID, accepted workload UID and boot ID fence its calls, so the +controller does not assign Gateway connection ownership. ## Launch @@ -313,7 +314,7 @@ or a Session's setup is still running, so the suspend never interrupts it; a del This hold has `status.operatingMode: Suspended` with no stop stage. When the hold ends, the controller publishes `Quiescing` with the attempt before its Quiesce RPC. The Gateway admits that RPC from the controller through an informer snapshot that still shows the held suspend, but denies other Runner -proxy methods there; incarnation, Pod readiness, connection epoch, and caller checks still apply. +proxy methods there; incarnation, Pod readiness and caller checks still apply. While a scheduled Pod is Pending, the reason is `PullingImage` when its latest Event is an image pull, and the message carries that latest Event; when no Event can be read the message says only that the Pod is not running yet. diff --git a/agentplane/sandbox_controller/internal/controller/BUILD.bazel b/agentplane/sandbox_controller/internal/controller/BUILD.bazel index 1a33f94c455..2ea3833a95b 100644 --- a/agentplane/sandbox_controller/internal/controller/BUILD.bazel +++ b/agentplane/sandbox_controller/internal/controller/BUILD.bazel @@ -7,7 +7,6 @@ go_library( "behavior.go", "cache.go", "claim.go", - "control_service.go", "controller.go", "environment_runtime_credential.go", "firecracker.go", diff --git a/agentplane/sandbox_controller/internal/controller/apply.go b/agentplane/sandbox_controller/internal/controller/apply.go index ce51b5cb1ca..8f4437bae24 100644 --- a/agentplane/sandbox_controller/internal/controller/apply.go +++ b/agentplane/sandbox_controller/internal/controller/apply.go @@ -38,12 +38,6 @@ func (r *Reconciler) applyOwned(ctx context.Context, sandbox *sandboxapi.Sandbox return nil, err } owned := metav1.IsControlledBy(current, sandbox) - // A control Service created by the Gateway has a non-controller Sandbox reference. - if current.GetKind() == "Service" && metav1.GetControllerOf(current) == nil { - for _, owner := range current.GetOwnerReferences() { - owned = owned || (owner.APIVersion == sandboxapi.GroupVersion.String() && owner.Kind == "Sandbox" && owner.Name == sandbox.Name && owner.UID == sandbox.UID) - } - } if !owned || !current.GetDeletionTimestamp().IsZero() { return &unready{reason: conflictReason, message: fmt.Sprintf("%s %s is foreign or being deleted", current.GetKind(), current.GetName())}, nil } diff --git a/agentplane/sandbox_controller/internal/controller/cache.go b/agentplane/sandbox_controller/internal/controller/cache.go index a1219c85fe5..0113b52c76c 100644 --- a/agentplane/sandbox_controller/internal/controller/cache.go +++ b/agentplane/sandbox_controller/internal/controller/cache.go @@ -13,7 +13,7 @@ import ( ) // CacheByObject restricts the manager's cache. Of the kinds the controller creates or watches by the -// Sandbox label (Pods, PVCs, ConfigMaps, Services and, with KubeVirt, VMs and VMIs) it holds only +// Sandbox label (Pods, PVCs, ConfigMaps and, with KubeVirt, VMs and VMIs) it holds only // objects carrying the label, so an unrelated object in the namespace is neither stored nor enqueued. // Events carry no labels: it holds those about any Pod. It holds every Identity and // WorkloadCredential in the namespace, watched by identity or owner reference. @@ -29,7 +29,6 @@ func CacheByObject(kubeVirt bool) (map[client.Object]cache.ByObject, error) { &corev1.Pod{}: {Label: ours}, &corev1.PersistentVolumeClaim{}: {Label: ours}, &corev1.ConfigMap{}: {Label: ours}, - &corev1.Service{}: {Label: ours}, &corev1.Event{}: {Field: fields.OneTermEqualSelector("involvedObject.kind", "Pod")}, } if kubeVirt { diff --git a/agentplane/sandbox_controller/internal/controller/children_integration_test.go b/agentplane/sandbox_controller/internal/controller/children_integration_test.go index cd491d63fda..96891231035 100644 --- a/agentplane/sandbox_controller/internal/controller/children_integration_test.go +++ b/agentplane/sandbox_controller/internal/controller/children_integration_test.go @@ -161,7 +161,7 @@ func TestManagerCacheSyncNeedsOnlyGrantedSandboxWatchKinds(t *testing.T) { if _, err := kube.Clientset.RbacV1().Roles(ns).Create(ctx, &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "sandbox-controller"}, Rules: []rbacv1.PolicyRule{ - {APIGroups: []string{""}, Resources: []string{"pods", "persistentvolumeclaims", "configmaps", "services", "events"}, Verbs: []string{"get", "list", "watch"}}, + {APIGroups: []string{""}, Resources: []string{"pods", "persistentvolumeclaims", "configmaps", "events"}, Verbs: []string{"get", "list", "watch"}}, {APIGroups: []string{sandboxapi.GroupVersion.Group}, Resources: []string{"sandboxes", "sandboxtemplates"}, Verbs: []string{"get", "list", "watch"}}, {APIGroups: []string{policyapi.SchemeGroupVersion.Group}, Resources: []string{"identities", "workloadcredentials"}, Verbs: []string{"get", "list", "watch"}}, }, @@ -226,10 +226,7 @@ func TestManagerCacheSyncNeedsOnlyGrantedSandboxWatchKinds(t *testing.T) { // Exercise a real API server without KubeVirt and real SSA field ownership and manager watches. func TestChildApplyOwnershipAndDriftWithoutKubeVirt(t *testing.T) { ctrl.SetLogger(logr.FromSlogHandler(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) - // Kubernetes defaults selector-less headless Services to dual-stack. Configure both - // families so their defaulted fields also pass validation on later drift updates. kube, err := kubeapiserver.Start( - kubeapiserver.WithAPIServerFlag("service-cluster-ip-range", "10.0.0.0/24,fd00::/108"), withRunnerCertificateCRD(t), ) if err != nil { @@ -401,23 +398,11 @@ func TestChildApplyOwnershipAndDriftWithoutKubeVirt(t *testing.T) { eventually(t, func() bool { return objects.Get(ctx, launchKey, &launch) == nil && launch.UID != deletedUID && launch.Data[LaunchFile] == wantLaunch }) - var service corev1.Service - serviceKey := types.NamespacedName{Namespace: ns, Name: key.Name + sandboxapi.ControlServiceSuffix} - if err := objects.Get(ctx, serviceKey, &service); err != nil { - t.Fatal(err) - } - if err := objects.Patch(ctx, &service, client.RawPatch(types.MergePatchType, []byte(`{"spec":{"selector":{"wrong":"pod"},"ports":[{"port":8080,"targetPort":1234,"protocol":"TCP","name":"wrong"}]}}`))); err != nil { - t.Fatal(err) - } - eventually(t, func() bool { - return objects.Get(ctx, serviceKey, &service) == nil && len(service.Spec.Selector) == 0 && len(service.Spec.Ports) == 1 && service.Spec.Ports[0].Name == sandboxapi.ControlServicePortName && service.Spec.Ports[0].TargetPort.IntVal == sandboxapi.ControlServicePort - }) for _, test := range []struct { name, reason string foreign client.Object }{ {"foreign-launch", "ConfigMapConflict", &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foreign-launch-launch", Namespace: ns}}}, - {"foreign-service", "ServiceConflict", &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "foreign-service-control", Namespace: ns}, Spec: corev1.ServiceSpec{ClusterIP: corev1.ClusterIPNone}}}, } { t.Run(test.name, func(t *testing.T) { if err := objects.Create(ctx, test.foreign); err != nil { diff --git a/agentplane/sandbox_controller/internal/controller/control_service.go b/agentplane/sandbox_controller/internal/controller/control_service.go deleted file mode 100644 index 598520a8c7c..00000000000 --- a/agentplane/sandbox_controller/internal/controller/control_service.go +++ /dev/null @@ -1,27 +0,0 @@ -package sandboxcontroller - -import ( - "context" - - sandboxapi "github.com/agentydragon/ducktape/agentplane/sandboxes/api" - - corev1 "k8s.io/api/core/v1" -) - -// ensureControlService keeps the headless, selector-less Service whose EndpointSlice the Gateway owns. -func (r *Reconciler) ensureControlService(ctx context.Context, sandbox *sandboxapi.Sandbox) (*unready, error) { - return r.applyOwned(ctx, sandbox, map[string]any{ - "apiVersion": "v1", "kind": "Service", - "metadata": map[string]any{ - "name": sandbox.Name + sandboxapi.ControlServiceSuffix, "namespace": sandbox.Namespace, - "labels": map[string]string{sandboxapi.SandboxLabel: sandbox.Name}, - }, - "spec": map[string]any{ - "clusterIP": corev1.ClusterIPNone, "selector": nil, - "ports": []any{map[string]any{ - "name": sandboxapi.ControlServicePortName, "port": sandboxapi.ControlServicePort, - "protocol": "TCP", "targetPort": sandboxapi.ControlServicePort, - }}, - }, - }, "ServiceConflict") -} diff --git a/agentplane/sandbox_controller/internal/controller/controller.go b/agentplane/sandbox_controller/internal/controller/controller.go index 8eb2003e358..77e3e2615ca 100644 --- a/agentplane/sandbox_controller/internal/controller/controller.go +++ b/agentplane/sandbox_controller/internal/controller/controller.go @@ -185,11 +185,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl if identityGate != nil { gate = identityGate } - if serviceGate, err := r.ensureControlService(ctx, &sandbox); err != nil { - return ctrl.Result{}, err - } else if gate == nil { - gate = serviceGate - } if launchGate, err := r.ensureLaunch(ctx, &sandbox); err != nil { return ctrl.Result{}, err } else if gate == nil { @@ -987,7 +982,6 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { setup := ctrl.NewControllerManagedBy(mgr). For(&sandboxapi.Sandbox{}). Owns(&corev1.ConfigMap{}). - Owns(&corev1.Service{}). // A virt-launcher Pod is its VMI's, not the Sandbox's, so Pods map back by the label. Watches(&corev1.Pod{}, bySandboxLabel). Watches(&corev1.PersistentVolumeClaim{}, bySandboxLabel). diff --git a/agentplane/sandbox_controller/internal/controller/firecracker.go b/agentplane/sandbox_controller/internal/controller/firecracker.go index 3e8b83237fa..4f5df3ae003 100644 --- a/agentplane/sandbox_controller/internal/controller/firecracker.go +++ b/agentplane/sandbox_controller/internal/controller/firecracker.go @@ -152,7 +152,7 @@ func (r *Reconciler) convergeFirecrackerIntent(ctx context.Context, sandbox *san return blocked, nil } // DownwardAPI projections can coalesce intent changes. Complete the checkpoint - // before resuming so the recorded connection epoch always requires a real reconnect. + // before resuming so the restored Runtime requires a fresh observation. if state.Phase != "suspended" { return &unready{reason: "FirecrackerPending", message: "waiting for the pending suspension before resuming", retryAfter: 2 * time.Second}, nil } diff --git a/agentplane/sandbox_controller/internal/controller/identity_test.go b/agentplane/sandbox_controller/internal/controller/identity_test.go index b4ad92cdc7e..55404fc657b 100644 --- a/agentplane/sandbox_controller/internal/controller/identity_test.go +++ b/agentplane/sandbox_controller/internal/controller/identity_test.go @@ -29,37 +29,6 @@ func getBinding(t *testing.T, c client.Client) *policyapi.PolicyBinding { return &binding } -func TestSandboxOwnsAControlServiceBeforeAnyRunnerConnects(t *testing.T) { - r, c := newReconciler(t, true, testTemplate("test-image:v1")) - reconcileOnce(t, r) - var service corev1.Service - if err := c.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: sandboxName + sandboxapi.ControlServiceSuffix}, &service); err != nil { - t.Fatal(err) - } - if !metav1.IsControlledBy(&service, getSandbox(t, c)) { - t.Errorf("Service owner references = %v", service.OwnerReferences) - } - if service.Spec.ClusterIP != corev1.ClusterIPNone || len(service.Spec.Selector) != 0 { - t.Errorf("Service spec = %v, want headless without a selector", service.Spec) - } -} - -func TestControlServiceTheGatewayCreatedIsReconciled(t *testing.T) { - created := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, Name: sandboxName + sandboxapi.ControlServiceSuffix, - OwnerReferences: []metav1.OwnerReference{{APIVersion: sandboxapi.GroupVersion.String(), Kind: sandboxapi.SandboxKind, Name: sandboxName, UID: "uid-sandbox"}}, - }, Spec: corev1.ServiceSpec{ClusterIP: corev1.ClusterIPNone}} - r, c := newReconciler(t, true, testTemplate("test-image:v1"), created) - reconcileOnce(t, r) - var service corev1.Service - if err := c.Get(context.Background(), client.ObjectKeyFromObject(created), &service); err != nil { - t.Fatal(err) - } - if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != sandboxapi.ControlServicePort { - t.Errorf("Service ports = %v, want the control port", service.Spec.Ports) - } -} - const testIdentityUID = "00000000-0000-4000-8000-00000000000a" func testIdentity(name string, uid types.UID) *policyapi.Identity { diff --git a/agentplane/sandboxes/api/constants.go b/agentplane/sandboxes/api/constants.go index 02bd5bfa319..12c077ae014 100644 --- a/agentplane/sandboxes/api/constants.go +++ b/agentplane/sandboxes/api/constants.go @@ -27,10 +27,8 @@ const ( RunnerProxySandboxOwnerNameMetadataKey = "x-agentplane-sandbox-owner-name" RunnerProxyIncarnationUIDMetadataKey = "x-agentplane-incarnation-uid" - ControlServiceSuffix = "-control" - ControlServicePortName = "grpc" - ControlServicePort = 8080 - RunnerGatewayService = "runner-gateway" + ControlServicePort = 8080 + RunnerGatewayService = "runner-gateway" // EnvironmentRuntimeServicePort is the runner's mTLS listener port in a Pod or KubeVirt guest. EnvironmentRuntimeServicePort = 9090 EnvironmentRuntimeCredentialMetadataKey = "agentplane.allegedly.works/environment-runtime-credential" diff --git a/agentplane/system_tests/driver/central_sampler.py b/agentplane/system_tests/driver/central_sampler.py index 3b26adf094d..2ce5de52cb4 100644 --- a/agentplane/system_tests/driver/central_sampler.py +++ b/agentplane/system_tests/driver/central_sampler.py @@ -240,7 +240,7 @@ async def central_stack(cluster: Cluster, directory: Path, *, template: str) -> gateway_target = await _address(cluster, core, "runner-gateway", 8080) sampler_token = await _bound_token(cluster, core, "runner-gateway", gateway_pod, SERVICE) server_credentials = await service_credentials(cluster) - # Sandbox Ready proves Controller observation. Service endpoint readiness can precede + # Sandbox Ready proves Controller observation. A claimed EndpointSlice can precede # the Gateway's first snapshot, so prove the live read route before starting setup. gateway_token = directory / "gateway-reader-token" gateway_token.write_text(await _bound_token(cluster, core, "app", app, GATEWAY_AUDIENCE)) diff --git a/agentplane/system_tests/driver/sandbox.py b/agentplane/system_tests/driver/sandbox.py index 7902a17ba5a..f97ab96acdd 100644 --- a/agentplane/system_tests/driver/sandbox.py +++ b/agentplane/system_tests/driver/sandbox.py @@ -34,7 +34,6 @@ POD_TEMPLATE = "environment-tools" ENVIRONMENT_RUNTIME_CONTAINER = "environment-runtime" -ENVIRONMENT_RUNTIME_LEASE_SUFFIX = "-environment-runtime" VM_TEMPLATE = "environment-vm" VM_WORKING_DIRECTORY = "/state/workspaces/accept" SANDBOX_READY_SECONDS = 300.0 @@ -396,15 +395,13 @@ async def kubectl(*args: str) -> dict[str, Any]: except Exception as error: return {"error": repr(error)} - resource, lease, pods, controller_logs, gateway_logs = await asyncio.gather( + resource, pods, controller_logs, gateway_logs = await asyncio.gather( kubectl("get", "sandbox", name, "-o", "json"), - kubectl("get", "lease", f"{name}{ENVIRONMENT_RUNTIME_LEASE_SUFFIX}", "-o", "json"), kubectl("get", "pods", "-l", f"{GROUP}/sandbox={name}", "-o", "json"), kubectl("logs", "deployment/sandbox-controller", "--all-pods=true", "--tail=100", "--since=10m"), kubectl("logs", "deployment/runner-gateway", "--all-pods=true", "--tail=100", "--since=10m"), ) diagnostic["kubernetes_sandbox"] = resource - diagnostic["environment_runtime_lease"] = lease diagnostic["sandbox_pods"] = pods diagnostic["sandbox_controller_logs"] = controller_logs diagnostic["runner_gateway_logs"] = gateway_logs diff --git a/agentplane/system_tests/scenarios/dispatcher.py b/agentplane/system_tests/scenarios/dispatcher.py index 62caee01b9d..36821fc0ca2 100644 --- a/agentplane/system_tests/scenarios/dispatcher.py +++ b/agentplane/system_tests/scenarios/dispatcher.py @@ -32,11 +32,7 @@ forward as _forward, ) from agentplane.system_tests.driver.identity import LAUNCH_DEFAULT_POLICY_SETS, scenario_identity -from agentplane.system_tests.driver.sandbox import ( - ENVIRONMENT_RUNTIME_CONTAINER, - ENVIRONMENT_RUNTIME_LEASE_SUFFIX, - POD_TEMPLATE, -) +from agentplane.system_tests.driver.sandbox import ENVIRONMENT_RUNTIME_CONTAINER, POD_TEMPLATE from agentplane.system_tests.driver.tls import service_credentials from agentplane.system_tests.driver.tracing import tracer from devinfra.testing.undeclared_outputs import undeclared_outputs_dir @@ -216,7 +212,6 @@ def persist() -> None: persist() commands = { "sandbox": ("get", "sandbox", name, "-o", "json"), - "environment_runtime_lease": ("get", "lease", f"{name}{ENVIRONMENT_RUNTIME_LEASE_SUFFIX}", "-o", "json"), "pod": ("get", "pod", name, "-o", "json"), "environment_runtime_log": ( "logs", diff --git a/cluster/cdk8s/agentplane/sandbox_controller.py b/cluster/cdk8s/agentplane/sandbox_controller.py index 8dbd15fb519..d9831131e05 100644 --- a/cluster/cdk8s/agentplane/sandbox_controller.py +++ b/cluster/cdk8s/agentplane/sandbox_controller.py @@ -155,7 +155,8 @@ def __init__( resources=[custom_resource("kubevirt.io", "virtualmachineinstances")], verbs=["get", "list", "watch"], ), - # Each Sandbox's control Service, which the Runner Gateway points at the connected runner. + # CLEANUP(added 2026-11-14): Remove after the Controller image that stops + # creating per-Sandbox control Services is pinned and all old Pods have left. RolePolicyRule( resources=[custom_resource("", "services")], verbs=["get", "list", "watch", "create", "patch"] ), From b7852af07522f0b7db01627536b0e7aa3c02ce90 Mon Sep 17 00:00:00 2001 From: agentydragon-agent <260791330+agentydragon-agent@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:19:11 -0800 Subject: [PATCH 4/5] Align architecture references with independent Runtime connections --- .../docs/adr_sandbox_session_control.md | 70 +++++++++++-------- .../runner_delivery_and_harness_design.md | 41 ++++++----- agentplane/docs/system_overview.md | 6 +- agentplane/plans/architecture_overview.md | 67 +++++++++--------- agentplane/plans/kubernetes_decoupling.md | 37 +++++----- agentplane/plans/task_dag.md | 2 +- .../runner_gateway/dispatcher/README.md | 7 +- .../environmentruntimeclient/client.go | 4 +- 8 files changed, 127 insertions(+), 107 deletions(-) diff --git a/agentplane/docs/adr_sandbox_session_control.md b/agentplane/docs/adr_sandbox_session_control.md index a391e7785d8..391e3cf6721 100644 --- a/agentplane/docs/adr_sandbox_session_control.md +++ b/agentplane/docs/adr_sandbox_session_control.md @@ -8,17 +8,28 @@ Former logical `OpenSession`/opening APIs have no compatibility path. The distin `Executor.OpenSession` remains an execution-connection handshake. References to the old logical name below preserve this ADR's historical decisions and examples. +**Gateway routing supersession:** the current contract is in [runner control](runner_control.md) and +the [Runner Gateway README](../runner_gateway/README.md). The leader-elected Controller binds the +Sandbox UID and accepted workload Pod UID through direct `WatchState(observe_only=true)` and keeps +lifecycle facts in memory. Callers use the stable `runner-gateway` Service; each Gateway replica +opens an independent zero-epoch watch, dials the Runtime at its Pod IP, and can call using Sandbox +UID, workload Pod UID and Runtime boot ID. Per-Sandbox control-Service and EndpointSlice creation +and Gateway connection-Lease publication are removed; old route objects may remain inert while +operational rollout holds are open. Dispatcher delivery Leases are unchanged. The Runtime receiver +still tolerates legacy nonzero epochs during the rollback window, with cleanup gated on pinning. D1 +and the Lease-dependent routing and signaling details below preserve the superseded design as history. + Scope: the responsibilities split across the services and custom resources that run a Sandbox (Pod -or VM) and deliver work to the Sessions in it. This covers the control Service, readiness and -runner health, quiesce, suspension, the Session message queue, and where each fact lives. It does +or VM) and deliver work to the Sessions in it. This covers runner routing, readiness and runner +health, quiesce, suspension, the Session message queue, and where each fact lives. It does not redesign storage, the History Service, or policy. ## What has to happen 1. **Run a Sandbox.** Reconcile the `Sandbox` CR into a state volume and, while running, one Pod or VM. Kind (`pod` or `vm`) comes from the template. -2. **Reach its runner.** Callers address `-control`; the replica holding the runner's - observation Lease answers. +2. **Reach its runner.** Callers use the stable `runner-gateway` Service; any replica with a fresh + observation of the current Runtime can answer. 3. **Say what state it is in.** Created, starting, booting, connected, ready, quiescing, suspended, tearing down, and why it is stuck when it is. 4. **Quiesce** (today's "seal"; one operation, names in D3). Before a Pod or VM goes away (suspend or delete), stop admitting work, @@ -35,11 +46,14 @@ not redesign storage, the History Service, or policy. ## Current state - **Sandbox Controller** (Go, `sandbox_controller/`): state PVC, Pod or VM, ServiceAccount, `PolicyBinding`, launch - ConfigMap, finalizer, and the `{sandbox}-control` Service. It drives quiesce: it calls `Runner.Quiesce` through the - gateway's runner proxy and records `status.quiescedIncarnation` on any quiesce and sets `Quiesced` on a deleting Sandbox only (`finalize`), beside the older sealed names. It - observes Runtime facts directly. `Ready` includes `WorkloadReady`, `EnvironmentRuntimeConnected` and `Bootstrapped`. `QuiesceResponse.stage` has + ConfigMap and finalizer. It directly observes each accepted Runtime with `WatchState(observe_only=true)`; + its lifecycle facts are in memory, fenced by Sandbox UID, accepted workload Pod UID, boot ID and + local stream generation. It drives quiesce: it calls `Runner.Quiesce` through the + gateway's runner proxy, records `status.quiescedIncarnation` on quiesce, and sets `Quiesced` on a + deleting Sandbox only (`finalize`), alongside the older sealed names. `Ready` includes + `WorkloadReady`, `EnvironmentRuntimeConnected` and `Bootstrapped`. `QuiesceResponse.stage` has one reader, the controller. -- **Runner Gateway** (`runner_gateway/`): state observation and proxy of the runner's private `Runner` service. It writes per-Sandbox Leases and EndpointSlices; it does not create the `-control` Service (its `services` RBAC rule stays until the controller's image is pinned). logical Session records and durable commands belong to the Dispatcher; the Gateway owns no Session registry or database. +- **Runner Gateway** (`runner_gateway/`): each replica independently observes and proxies the runner's private `Runner` service. The stable `runner-gateway` Service routes callers to any replica; there is no per-Sandbox control Service, Gateway connection Lease or EndpointSlice publication. Logical Session records and durable commands belong to the Dispatcher; the Gateway owns no Session registry or database. - **Integration app** (`app/`): submits Sandbox desired state and operator-owned grants through the Kubernetes API, reads Sandbox views, and attaches the output-only VM console. It acts as the requesting operator; the Sandbox Controller owns status and reconciled resources. @@ -52,7 +66,7 @@ not redesign storage, the History Service, or policy. The current sender path is queued: the app and Notification Service submit per-operation commands to Session Dispatcher, which durably records them and delivers after Sandbox readiness. The app requests Sandbox lifecycle changes through Kubernetes; the runner restores retained native state as it starts. -Live Runner reads use the UID/owner-fenced proxy through the per-Sandbox control Service. See the +Live Runner reads use the UID/owner-fenced proxy through the stable `runner-gateway` Service. See the [service-boundary contract](service_boundaries.md) and [Runner Gateway README](../runner_gateway/README.md) for current callers and authorization gates. @@ -80,28 +94,26 @@ flowchart LR K --> KDB[(Dispatcher database)] K -- writes spec.running, wakeGeneration on send --> CR K -- watches Ready and incarnation --> CR - K -- Runner calls through proxy --> GW[Runner Gateway] + K -- Runner proxy calls via stable Service --> GW[Runner Gateway] K -. placement when caller omits Sandbox .-> HS[History Service] App[App] -- creates, deletes, suspends, resumes --> CR[(Sandbox CR)] SC[Sandbox Controller] -- reconciles --> CR SC -- Quiesce through proxy --> GW - GW -- Lease, EndpointSlice --> L[(Leases)] - SC -- reads connectivity --> L - Runner -- control stream --> GW + SC -- direct observe_only WatchState --> Runner[Runtime] + GW -- independent zero-epoch WatchState and fenced calls --> Runner Runner -- journal push --> HS - Runner -- RunnerHeartbeat with facts --> GW - SC -- idle timer from idleSince --> L + Runner -- complete state snapshots --> SC ``` -| Component | Owns | Does not own | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| Sandbox CR | Desired state (`spec.running`, template), current facts (`status`: incarnation, `quiescedIncarnation`, conditions) | Anything per Session | -| Sandbox Controller | Pod or VM, state volume, identity, the `-control` Service, quiesce sequencing, idle suspension, deletion, folding every layer's fact into conditions | Runner connection, Session records, queues | -| Runner Gateway | The runner connection and its authentication, the Lease and EndpointSlice, the proxy of `Runner`, per-method authorization | Any Kubernetes object besides those two, Sessions, queues | -| Runner | Harnesses, journal, admission, quiesce mechanics, idle state | Policy about when to suspend | -| History Service | Event log, fold, Session directory including the writing Sandbox | Routing, waking, suspending | -| Session Dispatcher | The durable queue of messages and Session creations, wake-on-send, send-side authorization (who may wake), delivery states | Creating or deleting Sandboxes (it reads them and patches only `spec.running` and `spec.wakeGeneration` to wake), idle policy, history | -| App | Sandbox create, delete, suspend and resume and PolicyBinding (egress-grant) writes, through the Kubernetes API, with their checks; the VM console | Conditions, quiesce, idle policy, delivery | +| Component | Owns | Does not own | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Sandbox CR | Desired state (`spec.running`, template), current facts (`status`: incarnation, `quiescedIncarnation`, conditions) | Anything per Session | +| Sandbox Controller | Pod or VM, state volume, identity, direct Runtime observation, quiesce sequencing, idle suspension, deletion, folding observed facts into conditions | Gateway connection, Session records, queues | +| Runner Gateway | Independent Runtime observations and fenced `Runner` proxy calls, per-method authorization | Per-Sandbox Services, Leases and EndpointSlices, Sessions, queues | +| Runner | Harnesses, journal, admission, quiesce mechanics, idle state | Policy about when to suspend | +| History Service | Event log, fold, Session directory including the writing Sandbox | Routing, waking, suspending | +| Session Dispatcher | The durable queue of messages and Session creations, wake-on-send, send-side authorization (who may wake), delivery states | Creating or deleting Sandboxes (it reads them and patches only `spec.running` and `spec.wakeGeneration` to wake), idle policy, history | +| App | Sandbox create, delete, suspend and resume and PolicyBinding (egress-grant) writes, through the Kubernetes API, with their checks; the VM console | Conditions, quiesce, idle policy, delivery | "Session Dispatcher" names the queue service; it is not a controller. Idle is a Sandbox-level fact (every Session in the Sandbox idle), which is why its policy sits on the Sandbox. There is no Sandbox Service: requesters write `spec` through one @@ -129,7 +141,7 @@ Facts have different truth sources, so each gets its own condition and `Ready` i | ----------------------------- | ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | | `Running` | controller (Pod or VMI) | an incarnation exists and runs | | `WorkloadReady` | controller (Pod Ready) | the incarnation's Pod is Ready | -| `EnvironmentRuntimeConnected` | gateway (connection Lease) | a runner of this incarnation holds a live connection | +| `EnvironmentRuntimeConnected` | Controller (direct observe-only `WatchState`) | a fresh snapshot is observed from the Runtime of this incarnation | | `Bootstrapped` | runner (bootstrap result, republished at each connect) | the Sandbox's bootstrap ran to success; Sandbox-lifetime; False with reason `BootstrapFailed` if it failed or was interrupted | | `Ready` | controller | all of the above for the current incarnation; reason `BootstrapFailed` if bootstrap failed, `SuspendPending` while a suspend waits (D3) | | `Quiesced` | controller, from the runner's directly observed quiesce stage | False with reason `Draining`, then `Flushing`; True once every final cursor is acknowledged | @@ -150,9 +162,9 @@ one actor: the controller writes them all; every other layer publishes a fact th changes and periodically. Facts stay in memory, fenced by Sandbox UID, Pod UID, boot ID and local stream generation. Leadership takeover and reconnect start fresh observation and idle timers. Stale facts cannot establish connectivity, idleness or quiescence; persisted quiesce attempts and -terminal bootstrap failures survive leader changes. See [runner control](runner_control.md). -The Gateway retains its connection Lease for proxy ownership until the separate protocol removal; -it publishes no lifecycle facts there. +terminal bootstrap failures survive leader changes. See [runner control](runner_control.md). Each +Gateway replica observes independently and dials the current Runtime directly; it does not use or +publish a connection Lease. ## D3. Quiescing is one operation; names say which stage a Sandbox is in @@ -431,7 +443,7 @@ admission. The controller calls `Quiesce`; reads remain separately authorized. | Fact | Home | | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | | Desired running, template, incarnations, conditions, `quiescedIncarnation`, the controller's last `quiesceAttempt`, the rendered spec of each incarnation | Sandbox CR (Kubernetes) | -| Runner connected, boot state, `idleSince`, quiesce stage | Lease annotations (Kubernetes, Sandbox-level) | +| Runner connected, boot state, `idleSince`, quiesce stage | Controller memory from direct `WatchState` | | Session events, directory, writing Sandbox, summaries | History Service database | | Queued messages, their delivery state, queued Session creations | Dispatcher database | | Session journal and cursors | Runner volume, mirrored by the ingress | diff --git a/agentplane/docs/runner_delivery_and_harness_design.md b/agentplane/docs/runner_delivery_and_harness_design.md index bab80306d0d..264e4bff05e 100644 --- a/agentplane/docs/runner_delivery_and_harness_design.md +++ b/agentplane/docs/runner_delivery_and_harness_design.md @@ -30,24 +30,27 @@ The current runner contracts remain in [Session layering](thread_layering.md), t The app and Notification service create Sessions and submit commands through the Session Dispatcher. It records openings and commands in its own PostgreSQL database, groups delivery by Sandbox UID, and -uses per-Sandbox Kubernetes Leases plus database generations to fence replica ownership. It watches -Sandbox readiness and incarnation changes, then calls the Runner service through the Sandbox's -control Service and the Runner Gateway's proxy. It has no History Service client: routing and queue -recovery use the requested Sandbox, Kubernetes state, and runner control RPCs. See +uses a per-Sandbox delivery Lease plus database generations to fence delivery scheduling. It watches +Sandbox readiness and incarnation changes, then calls the Runner service through the stable +`runner-gateway` Service and its proxy. It has no History Service client: routing and queue recovery +use the requested Sandbox, Kubernetes state, and runner control RPCs. See [`delivery.go`](../runner_gateway/dispatcher/delivery.go), [`runner_client.go`](../runner_gateway/dispatcher/runner_client.go), and [`store.go`](../runner_gateway/dispatcher/store.go). -The Gateway connection role owns the runner's long-lived control connection and routes calls to the replica holding -that connection. The unified Go process also owns the Dispatcher queue through its separate delivery role. It does -not read the retired `sandbox_commands` Session table. The separate Gateway database resource is absent +Each Gateway replica independently opens a zero-epoch `WatchState` to the current Runtime, dialing it +directly at its Pod IP, and can serve calls after observing the Controller-bound Sandbox UID and +workload Pod UID. Runtime calls carry +those identities and the boot ID observed by that replica; the Runtime checks them against its bound +identity. The unified Go process also owns the Dispatcher queue through its separate delivery role. It +does not read the retired `sandbox_commands` Session table. The separate Gateway database resource is absent from current manifests. The runner journals admitted commands on its state volume. The History Service receives runner journals and serves retained events and folded Thread views, but these are separate service boundaries rather than a Dispatcher dependency: - **Dispatcher has no direct History dependency to remove.** Its current routing, queue recovery, - admission and delivery result paths use the requested Sandbox, Kubernetes state, leases and Runner + admission and delivery result paths use the requested Sandbox, Kubernetes state, delivery leases and Runner RPCs. No Dispatcher History client or call exists in the audited implementation. - **The app uses History for retained Thread data.** `EventLogStore` reads archived events and observations and validates the History Service's committed prefix; summaries come from the History @@ -71,8 +74,8 @@ There are two different Session inventories: - Runner `ListSessions` asks the current runner's `retained.Store` for persisted records and reports them with the current harness, setup, launch and activity state. The current - production callers are the app and Notification, through the per-Sandbox proxy with expected - Sandbox UID and owner. Dispatcher does not call the list RPC: an identical logical `CreateSession` + production callers are the app and Notification, through the stable `runner-gateway` Service with + expected Sandbox UID and owner. Dispatcher does not call the list RPC: an identical logical `CreateSession` retry returns its existing Session record without rediscovering a live runner attachment; a conflicting placement, spec or setup returns `ALREADY_EXISTS`. Runner test drivers use the read for lifecycle assertions. @@ -110,9 +113,11 @@ checks that the watched Sandbox still has that UID and that it owns the Sandbox' before continuing. The Go `ProxyRunner` sends the Sandbox name and UID to the proxy with expected-identity metadata. The proxy authenticates its caller and requires exact Sandbox UID and owner metadata, then resolves and checks the live Sandbox and ready Pod. -The control Service routes by the holder of that UID. This is a sequence of checks across separate -requests, so the reviewed contract should say where identity is bound and where it is revalidated, -including replacement between the Dispatcher check and proxy resolution. See +The stable `runner-gateway` Service may route to any replica. Each replica can serve only with its +own fresh Runtime observation; the proxy call carries Sandbox UID, current workload Pod UID and the +observed Runtime boot ID, which Runtime checks against the Controller-bound identity. These checks +span separate requests, so the reviewed contract should say where identity is bound and where it is +revalidated, including replacement between the Dispatcher check and proxy resolution. See [`proxy.go`](../runner_gateway/internal/gateway/proxy.go), [`sandboxes.go`](../runner_gateway/internal/gateway/sandboxes.go) with the UID fence in [`reader.go`](../sandboxes/sandboxreader/reader.go), and @@ -202,7 +207,7 @@ reply from direct runner admission is unknown: resolve or retry the same command runner storage. Loss of that storage ends runner-owned state; History is not a reconstruction source. Two choices remain open. A namespace/name plus mandatory UID request can use the existing -namespace-local Kubernetes read and control Service as a locator; a UID-only request needs an +namespace-local Kubernetes read as a locator; a UID-only request needs an authoritative UID-to-object resolver that works during suspension and rejects ambiguous or stale matches. Either representation must persist enough information to re-resolve the _same_ object and must never promote name-only discovery to an execution address. If the runner alone selects a store, @@ -230,9 +235,11 @@ its independent caller grant. Gateway observes Runtime through its private mTLS Dispatcher admission, explicit resume and automatic wake retain their distinct grants. The Dispatcher PostgreSQL queue, per-Sandbox delivery Lease and database-generation fence remain -separate from the connection Lease and EndpointSlice. A worker on replica A may deliver through -`-control` to a runner connection held by replica B; co-location does not eliminate that -routing or the queue. The service has no History client or separate Gateway registry/database. +the delivery ownership mechanism. Gateway routing no longer publishes or uses a connection Lease or +EndpointSlice; old route objects may remain inert while rollout holds are open. A worker calls the +stable `runner-gateway` Service, and any replica with a fresh observation can call the current +Runtime directly at its Pod IP using the Sandbox UID, workload Pod UID and boot ID. The service has +no History client or separate Gateway registry/database. Runner journal ingress and Quiesce seal acknowledgements belong to History; app and Notification also read History independently. See the [Runner Gateway README](../runner_gateway/README.md) for current runtime and deployment ownership, including migration ordering and Flux image automation. diff --git a/agentplane/docs/system_overview.md b/agentplane/docs/system_overview.md index dd11f9cc2e8..857b3a726e2 100644 --- a/agentplane/docs/system_overview.md +++ b/agentplane/docs/system_overview.md @@ -133,8 +133,8 @@ flowchart LR sidecar -->|OTLP/gRPC| alloy guest -.->|OTLP/gRPC telemetry route| relay relay -->|OTLP/gRPC| alloy - gw -->|mTLS Runner / Executor; bound-UID WatchState with zero or legacy epoch| runtime - gw -->|mTLS Runner / Executor; bound-UID WatchState with zero or legacy epoch| vmruntime + gw -->|mTLS Runner / Executor; zero-epoch WatchState; UID and boot fences| runtime + gw -->|mTLS Runner / Executor; zero-epoch WatchState; UID and boot fences| vmruntime runner -.->|History mTLS through CONNECT| sidecar sidecar -.->|opaque History TLS| egress egress -.->|opaque SessionJournalIngress.Push TLS| hs @@ -444,7 +444,7 @@ flowchart LR sessions -->|journal inventory| delivery end history[History ingress] - gateway -->|mTLS Runner / Executor; bound-UID zero-epoch calls and WatchState; legacy nonzero epochs| runtime + gateway -->|mTLS Runner / Executor; zero-epoch WatchState; UID and boot fences| runtime policy_api[Kubernetes PolicySets / PolicyBindings / Identities] gateway -->|list/watch grants; patch EnvironmentReady| policy_api sandboxctl[Sandbox Controller] -->|passive WatchState binds workload UID and observes facts: mTLS gRPC| runtime diff --git a/agentplane/plans/architecture_overview.md b/agentplane/plans/architecture_overview.md index 739fb4b0951..1f769e5a5b7 100644 --- a/agentplane/plans/architecture_overview.md +++ b/agentplane/plans/architecture_overview.md @@ -107,9 +107,11 @@ exploration (#10990, closed after this section absorbed it). ### Anchored to the cluster in every direction - **Workload objects.** The Sandbox Controller creates a Pod or KubeVirt VM, the state and workspace - PVCs, the launch ConfigMap, the ServiceAccount and the headless `-control` Service for - each Sandbox (`sandbox_controller/`); the Runner Gateway writes the Service's EndpointSlice. They - need scheduling, image pulls, kubelet mounts, cluster DNS and Cilium NetworkPolicy. + PVCs, the launch ConfigMap and the ServiceAccount for each Sandbox (`sandbox_controller/`). Runtime + proxy callers use the stable `runner-gateway` Service. Gateway replicas dial the Runtime at its + Pod IP; per-Sandbox control-Service and EndpointSlice creation and connection-Lease publication + have been removed, though old route objects may remain inert during rollout. These workloads need + scheduling, image pulls, kubelet mounts, cluster DNS and Cilium NetworkPolicy. - **Workload identity.** Pods get kubelet-projected tokens: 600 s in Sandbox Pods (`cluster/cdk8s/agentplane/sandbox_pod.py`), 3600 s in service Deployments. Destination services use the Go resolver, which reuses accepted TokenReviews for up to 30 s @@ -133,18 +135,18 @@ exploration (#10990, closed after this section absorbed it). ### Record machinery, and what D3 and D4 do with it -D1 and D2 keep all of it as it is. D4's costs are a relative judgment. +D1 and D2 keep the cluster record machinery. D4's costs are a relative judgment. -| Behavior | D3 | D4 | -| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Five CRD kinds with schema, defaults and CEL rules (idle-suspend consistency, name length, exactly one of Pod or VM), tested against a real apiserver (`crds/test_sandbox_admission.py`) | Same, with CRDs per plane | Port or drop each rule at the API edge; medium | -| Conflict-checked writes: exclusive create, `uid` and `resourceVersion` preconditions (`runner_gateway/environments/internal/management/access.go`, the Dispatcher's wake), server-side apply with one field manager per reader on `PolicyBinding` status (`policy/reader/readiness.go`). The idle-suspend wake race relies on `resourceVersion` being object-wide (`sandbox_controller/internal/controller/idle.go`) | Same | A version column; low | -| Status subresource with its own RBAC verb, conditions with `observedGeneration`, a per-reader condition on each `PolicyBinding` | Same | A separate write path and authorization for status; medium | -| List-then-watch: client-go informers and controller-runtime caches in the Go app and services (the Action Service, both controllers, the Gateway with the Dispatcher, egress) | Same, with a second client per plane | A resumable watch with a "gone, relist" signal; high, the one hard piece, since Postgres watches here send a snapshot then changes and cannot resume ([API aggregation](https://github.com/agentydragon/ducktape/blob/7e22be0335/agentplane/plans/api_aggregation_exploration.md)) | -| Leases: `-environment-runtime` fences proxy ownership by `leaseTransitions` and names the holding Gateway replica; lifecycle facts go directly from Runtime to Controller; `dispatcher-` beside a Postgres fence; leader election for both controllers | Move with the Sandbox family | Compare-and-set epoch with TTL and holder routing; medium. The Dispatcher's Postgres fence has no expiry, so it stops stale writers but cannot pick a live owner; the [proposed merge](https://github.com/agentydragon/ducktape/blob/7e22be0335/agentplane/plans/dispatcher_gateway_merge.md#one-owner-per-sandbox) makes the runner connection holder the owner | -| Ownership and deletion: the Sandbox owns every child, a finalizer holds deletion until the runner has quiesced, garbage collection removes the rest; the Gateway (`runner_gateway/internal/gateway/sandboxes.go`) and the relay policy identify a Pod by its owner chain | Ownership does not cross apiservers (general Kubernetes behavior, not tried): the controller deletes each child kind itself, and the Pod-to-Sandbox check needs another binding | A sweeper per child kind and a held deletion; medium to high | -| Provenance in the app: the Flux label `kustomize.toolkit.fluxcd.io/name` and Sandbox/Identity owner references classify displayed bindings (`app/frontend/bindings/table.tsx`); `managedFields` records writers | Same | A writer column; low to medium | -| Clients: `kubectl`, Flux, `kubernetes_asyncio` in four remaining non-test Python modules, client-go and controller-runtime in the Go services | Kept, with a second kubeconfig | Lost | +| Behavior | D3 | D4 | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Five CRD kinds with schema, defaults and CEL rules (idle-suspend consistency, name length, exactly one of Pod or VM), tested against a real apiserver (`crds/test_sandbox_admission.py`) | Same, with CRDs per plane | Port or drop each rule at the API edge; medium | +| Conflict-checked writes: exclusive create, `uid` and `resourceVersion` preconditions (`runner_gateway/environments/internal/management/access.go`, the Dispatcher's wake), server-side apply with one field manager per reader on `PolicyBinding` status (`policy/reader/readiness.go`). The idle-suspend wake race relies on `resourceVersion` being object-wide (`sandbox_controller/internal/controller/idle.go`) | Same | A version column; low | +| Status subresource with its own RBAC verb, conditions with `observedGeneration`, a per-reader condition on each `PolicyBinding` | Same | A separate write path and authorization for status; medium | +| List-then-watch: client-go informers and controller-runtime caches in the Go app and services (the Action Service, both controllers, the Gateway with the Dispatcher, egress) | Same, with a second client per plane | A resumable watch with a "gone, relist" signal; high, the one hard piece, since Postgres watches here send a snapshot then changes and cannot resume ([API aggregation](https://github.com/agentydragon/ducktape/blob/7e22be0335/agentplane/plans/api_aggregation_exploration.md)) | +| Leases: Runtime lifecycle facts reach the leader-elected Controller through direct observe-only `WatchState` and stay in memory; `dispatcher-` plus its Postgres generation fence owns delivery scheduling; leader election for both controllers | Move Dispatcher delivery ownership with the Sandbox family | A database-backed Dispatcher delivery Lease and generation fence; medium. Runtime routing needs no per-Sandbox connection Lease or holder epoch | +| Ownership and deletion: the Sandbox owns every child, a finalizer holds deletion until the runner has quiesced, garbage collection removes the rest; the Gateway (`runner_gateway/internal/gateway/sandboxes.go`) and the relay policy identify a Pod by its owner chain | Ownership does not cross apiservers (general Kubernetes behavior, not tried): the controller deletes each child kind itself, and the Pod-to-Sandbox check needs another binding | A sweeper per child kind and a held deletion; medium to high | +| Provenance in the app: the Flux label `kustomize.toolkit.fluxcd.io/name` and Sandbox/Identity owner references classify displayed bindings (`app/frontend/bindings/table.tsx`); `managedFields` records writers | Same | A writer column; low to medium | +| Clients: `kubectl`, Flux, `kubernetes_asyncio` in four remaining non-test Python modules, client-go and controller-runtime in the Go services | Kept, with a second kubeconfig | Lost | D3's apiserver would be [kine](https://github.com/k3s-io/kine) on the CNPG cluster, which per its README implements the etcd API subset the apiserver needs over SQLite, MySQL, Postgres, NATS or T4, @@ -157,10 +159,11 @@ control plane to run: certificates, HA, upgrades, backups, monitoring. A kind stays on the primary when it owns primary objects, is owned by them, or protects them. -- **The Sandbox family** (`Sandbox`, its `PolicyBinding`, its Leases) moves together: the binding and - Leases are owned only by the Sandbox, and the Policy Controller already keys native RoleBindings by - binding UID in an annotation. The family grows with Sandbox count and holds the hot writes; its - cost is its ownership of the workload children. +- **The Sandbox family** (`Sandbox`, its `PolicyBinding`, and Dispatcher delivery ownership) moves + together: the binding and workload children follow Sandbox ownership, while delivery ownership is + keyed by Sandbox UID. The Policy Controller already keys native RoleBindings by binding UID in an + annotation. The family grows with Sandbox count and holds the hot writes; its cost is its ownership + of the workload children. - **The catalog** (`PolicySet`, `EgressCredential`, `SandboxTemplate`, the static `PolicyBinding`s) owns nothing and is owned by nothing, so it moves cheaply, and gains nothing from moving except that a reader then watches one plane. @@ -259,14 +262,14 @@ make it restore and continue that history" (09:53). What each operation leaves: -| | Retire | Archive | Delete | -| --------------------------------------------------------------------------- | ---------------------------------- | --------------- | ------ | -| Sandbox CR and children (ServiceAccount, control Service, bindings, Leases) | Kept, stopped | Gone | Gone | -| State and workspace PVCs | Kept; the workspace can be dropped | Gone | Gone | -| Dispatcher queue rows | Kept | Gone | Gone | -| History transcript and app Thread | Kept | Kept, read-only | Gone | -| History's record of the Sandbox (namespace, name, UID, history tags) | Kept | Kept | Purged | -| Can run again | Yes | No | No | +| | Retire | Archive | Delete | +| -------------------------------------------------------------------- | ---------------------------------- | --------------- | ------ | +| Sandbox CR and workload children (ServiceAccount, bindings) | Kept, stopped | Gone | Gone | +| State and workspace PVCs | Kept; the workspace can be dropped | Gone | Gone | +| Dispatcher queue rows | Kept | Gone | Gone | +| History transcript and app Thread | Kept | Kept, read-only | Gone | +| History's record of the Sandbox (namespace, name, UID, history tags) | Kept | Kept | Purged | +| Can run again | Yes | No | No | Deleting a Sandbox today is an archive without History's record of the Sandbox. The operator (23:24): "history's record of a Sandbox would be an entity in History Service, and Sandbox CR could @@ -364,7 +367,7 @@ Held until one is chosen: `spec.kubeConfig` apply. - **The price of a retired Sandbox.** On `agentplane-testing`, stop one with `spec.wakeOnSend: false`, count its records and watcher cache entries, weigh its state PVC, and - confirm that neither of its Leases renews. Checks D1's arithmetic. + confirm the Dispatcher delivery Lease lapses once no work remains. Checks D1's arithmetic. - **D4's git glue without a provider of our own.** Whether a generic REST Terraform provider can drive a records API. Not tried. @@ -377,17 +380,17 @@ Held until one is chosen: history kept (2026-10-18 23:29), floated suspending instead of deleting, offered deleting the history with the Sandbox (2026-10-19 02:04), and wants the workspace disposable on its own while the state PVC and metadata stay with the history (09:42). -- **Sandbox children** (Pod or VM, state and workspace PVCs, ServiceAccount, `-control` - Service, the Sandbox's own `PolicyBinding`). Owned by the Sandbox and removed with it; both PVCs +- **Sandbox children** (Pod or VM, state and workspace PVCs, ServiceAccount, the Sandbox's own + `PolicyBinding`). Owned by the Sandbox and removed with it; both PVCs survive suspension. - **Grants and catalog** (`PolicySet`, static `PolicyBinding`s, `SandboxTemplate`, `EgressCredential`). Authored in git and applied by Flux; the Sandbox Controller reconciles the Sandbox-owned PolicyBinding from the Sandbox's selected PolicySets. Operator: wants the Flux-managed shared permission sets kept gitops-driven, and floated a second control plane "if it came to it" (2026-10-18 22:54, 23:02). - **Leases.** `dispatcher-`, renewed every 10 s (`LeaseRenewInterval`) while its - Sandbox has queued or delivering work and left to lapse otherwise (#11035), and `-runner` - from the Gateway while a runner is connected. The Dispatcher also keeps a Postgres - `sandbox_ownership` generation fence per Sandbox, with no expiry + Sandbox has queued or delivering work and left to lapse otherwise (#11035). Runtime lifecycle + facts come from direct Controller observation and stay in memory. The Dispatcher also keeps a + Postgres `sandbox_ownership` generation fence per Sandbox, with no expiry ([size today](#size-today)). ### Databases and runner storage diff --git a/agentplane/plans/kubernetes_decoupling.md b/agentplane/plans/kubernetes_decoupling.md index 0b5236b40b2..2393ddce34c 100644 --- a/agentplane/plans/kubernetes_decoupling.md +++ b/agentplane/plans/kubernetes_decoupling.md @@ -17,14 +17,14 @@ would reopen that baseline. It does not repeat the overview's record-machinery a Checked on `devel` `a2c0b74e3`. -| Coupling | Where | -| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Caller authentication | `kubernetes_bearer` TokenReview in History, Notifications, LLM ingress, the Action Service, the Runner Gateway and the app. Every SVID check makes one uncached GET of the `WorkloadCredential` and the `Identity` ([how a verifier resolves a peer](../docs/workload_authentication.md#how-a-verifier-resolves-a-peer)). cert-manager issues SVIDs from the `agentplane-workload` ClusterIssuer; the Policy Controller writes `WorkloadCredential` status. | -| Agentplane records as CRDs | CRDs in [`crds/manifests`](../crds/manifests): `Identity`, `PolicySet`, `PolicyBinding`, `WorkloadCredential`, `EgressCredential`, `SandboxTemplate`, `Sandbox`. The Action Service, egress and the MCP edge read them with their own informers ([`policy/reader`](../policy/reader)); only the app and browser use `PolicyService`'s watch RPCs. orchestration_v1 writes `Identity`, `PolicySet` and `PolicyBinding` through a kube client. CEL admission: `crds/test_*_admission.py`. | -| Records whose output is Kubernetes | `KubernetesRoleSet`, `ServiceAccountGrant`, `ManagedServiceAccount` and `PolicySet` `kubernetes_*` rules become RoleBindings for agents that run `kubectl`. Product behavior; stays on Kubernetes under every option. | -| Environments | The Sandbox Controller creates the Pod or KubeVirt VM (Firecracker runs inside a Pod), state and workspace PVCs, launch ConfigMap, ServiceAccount and `-control` Service. The Gateway dials the Runtime at its Pod IP ([runner control](../docs/runner_control.md)), fences calls by Sandbox UID and incarnation Pod UID, publishes Runtime facts on the `-environment-runtime` Lease and routes callers with the EndpointSlice. `WorkloadCredential` `spec.pod` and Pod-bound tokens tie identity to the Pod. Egress is a sidecar or relay plus Cilium policy. | -| Placement and ownership | The Dispatcher's ownership Lease ([`ownership.go`](../runner_gateway/dispatcher/ownership.go)); History, Notifications and the sampler already use [`postgres/lease`](../postgres/lease). | -| API addresses | `EnvironmentTarget` is namespace, name and UID; `SubmitInput` and `InterruptTurn` carry `expected_sandbox_uid`; `Dispatch` names a Sandbox; callers dial `{sandbox}-control.{namespace}.svc` and send `x-agentplane-sandbox*` headers. | +| Coupling | Where | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Caller authentication | `kubernetes_bearer` TokenReview in History, Notifications, LLM ingress, the Action Service, the Runner Gateway and the app. Every SVID check makes one uncached GET of the `WorkloadCredential` and the `Identity` ([how a verifier resolves a peer](../docs/workload_authentication.md#how-a-verifier-resolves-a-peer)). cert-manager issues SVIDs from the `agentplane-workload` ClusterIssuer; the Policy Controller writes `WorkloadCredential` status. | +| Agentplane records as CRDs | CRDs in [`crds/manifests`](../crds/manifests): `Identity`, `PolicySet`, `PolicyBinding`, `WorkloadCredential`, `EgressCredential`, `SandboxTemplate`, `Sandbox`. The Action Service, egress and the MCP edge read them with their own informers ([`policy/reader`](../policy/reader)); only the app and browser use `PolicyService`'s watch RPCs. orchestration_v1 writes `Identity`, `PolicySet` and `PolicyBinding` through a kube client. CEL admission: `crds/test_*_admission.py`. | +| Records whose output is Kubernetes | `KubernetesRoleSet`, `ServiceAccountGrant`, `ManagedServiceAccount` and `PolicySet` `kubernetes_*` rules become RoleBindings for agents that run `kubectl`. Product behavior; stays on Kubernetes under every option. | +| Environments | The Sandbox Controller creates the Pod or KubeVirt VM (Firecracker runs inside a Pod), state and workspace PVCs, launch ConfigMap and ServiceAccount. Its leader directly observes each accepted Runtime with `WatchState(observe_only=true)`; lifecycle facts stay in memory, fenced by Sandbox UID, accepted workload Pod UID and boot ID. Gateway replicas dial the Runtime directly at its Pod IP ([runner control](../docs/runner_control.md)); proxy callers use the stable `runner-gateway` Service. Runtime calls carry Sandbox UID, workload Pod UID and boot ID. Per-Sandbox control-Service and EndpointSlice creation and Gateway connection-Lease publication are removed; old route objects may remain inert through the operational rollout holds. `WorkloadCredential` `spec.pod` and Pod-bound tokens tie identity to the Pod. Egress is a sidecar or relay plus Cilium policy. | +| Placement and ownership | The Dispatcher's ownership Lease ([`ownership.go`](../runner_gateway/dispatcher/ownership.go)); History, Notifications and the sampler already use [`postgres/lease`](../postgres/lease). | +| API addresses | `EnvironmentTarget` is namespace, name and UID; `SubmitInput` and `InterruptTurn` carry `expected_sandbox_uid`; `Dispatch` names a Sandbox; Runtime proxy callers use the stable `runner-gateway` Service and send the `x-agentplane-sandbox*` identity headers. | ## What tests need today @@ -70,18 +70,15 @@ own: [`PRIVILEGED_BACKEND_AUTH_DESIGN`](task_dag.md#privileged_backend_auth_design--one-service-authentication-mode)), with credential resolution through the same reader. 3. Dispatcher, EnvironmentService and CLI addresses become opaque environment IDs. -4. The Gateway's per-Sandbox connection Lease and EndpointSlice are transitional routing records. - The selected contract lets any authenticated replica call the Runtime, which fences by Sandbox - UID, workload incarnation and boot ID from the Controller-bound identity; connection epochs do - not authorize calls. The direct Controller reader and Lease-fact removal landed in #16324 and - #16328, and the receiver binding/independent-watch compatibility landed in #16345. Stage A pins - that Runtime receiver and cycles older live staging Runtimes, then publishes Gateway routes that - retain the existing Lease/EndpointSlice only for old callers. Stage B switches caller defaults to - the stable `runner-gateway` Service after all Gateways run A. Stage C removes the old route records - after all B callers have rolled out. The [runtime state observation plan](runtime_state_observation.md) - and [task DAG](task_dag.md#runner_gateway_independent_runtime_connections--let-every-gateway-call-a-bound-runtime) - record these holds. A later compatibility cleanup removes unused RBAC and legacy receiver epoch - fields. The Dispatcher delivery Lease is a separate scheduling protocol and remains out of scope. +4. Gateway code supports independent zero-epoch Runtime watches and direct calls from every replica, + fenced by the Controller-bound Sandbox UID, workload Pod UID and Runtime boot ID. Each replica + dials the Runtime at its Pod IP; proxy callers use the stable `runner-gateway` Service. The + per-Sandbox control-Service and EndpointSlice creation and connection-Lease publication paths + have been removed, though old route objects may remain inert while operational rollout holds are + open. See the [runtime state observation plan](runtime_state_observation.md) and [task DAG](task_dag.md#runner_gateway_cutover_rollout--operational-holds-for-the-prepared-cutover) + for those holds. The Runtime receiver still tolerates legacy nonzero epochs until a later + pin-gated cleanup. The Dispatcher delivery Lease is a separate scheduling protocol and remains + unchanged. Service tests could then run against one fake `PolicyService` instead of envtest. The store stays an API server. diff --git a/agentplane/plans/task_dag.md b/agentplane/plans/task_dag.md index 3f23c647951..926551a5257 100644 --- a/agentplane/plans/task_dag.md +++ b/agentplane/plans/task_dag.md @@ -2468,7 +2468,7 @@ recurs, or close the incident follow-up if ordinary use shows no recurrence. Pre **In flight, 2026-11-14: independent Gateway observers, stable-Service caller routing, then route publication removal are prepared as separate PRs in this batch.** The #16345 Runtime receiver is -pinned as runner-disk `devel-20261114063459-d8258d6`; live Runtime provenance still needs operator +included in runner-disk `devel-20261114071615-9e1f8fe` on fresh devel; live Runtime provenance still needs operator verification before zero-epoch Gateway deployment. The [rollout plan](runtime_state_observation.md#migration-and-rollback) lists backend-specific inventory evidence. Firecracker suspend/resume preserves its Runtime process and cannot upgrade an older receiver; a retained old checkpoint keeps this gate closed diff --git a/agentplane/runner_gateway/dispatcher/README.md b/agentplane/runner_gateway/dispatcher/README.md index 47f2bb0ff18..002c1dfb0b9 100644 --- a/agentplane/runner_gateway/dispatcher/README.md +++ b/agentplane/runner_gateway/dispatcher/README.md @@ -7,7 +7,7 @@ shared TLS listener. The credential-holding Sandbox relay reaches that same list central-proxy CONNECT and presents its Runner SVID. Dispatcher and EnvironmentService methods retain their own caller rules, Dispatcher bearer audience and delegation policy. The worker delivers through the Runner proxy once -the exact Sandbox is ready, with per-Sandbox Lease and PostgreSQL generation fencing. +the exact Sandbox is ready, with a per-Sandbox delivery Lease and PostgreSQL generation fencing. The manual queue, replica and wake probes are documented in the [system tests](../../system_tests/README.md#dispatcher-queue-and-replicas). @@ -115,8 +115,9 @@ The manual queue, replica and wake probes are documented in the - **Replica ownership**: a per-Sandbox Lease assigns delivery scheduling ownership while the Sandbox has a `QUEUED_IN_DISPATCHER` or `DELIVERING_TO_RUNNER` Dispatch or Session creation. Its PostgreSQL generation fences stale claims, settlements and recovery. A Sandbox without such work is no longer - renewed and its Lease lapses; the next work reacquires it. The Runner Gateway's separate connection - Lease still owns the runner stream; Dispatcher scheduling does not affect it. A new owner requeues + renewed and its Lease lapses; the next work reacquires it. Gateway replicas independently observe + the Controller-bound Runtime and can serve calls through the stable `runner-gateway` Service; + Gateway routing does not use or publish a connection Lease. A new Dispatcher owner requeues only that Sandbox's in-flight work, preserving at-least-once redelivery with the same command IDs. - **Runner statuses**: `ABORTED` (quiescing) returns the command to the queue until the Sandbox has a new incarnation; `UNAVAILABLE` and deadline retry with backoff; diff --git a/agentplane/sandboxes/environmentruntimeclient/client.go b/agentplane/sandboxes/environmentruntimeclient/client.go index f24065c6545..b34fb9a2a0f 100644 --- a/agentplane/sandboxes/environmentruntimeclient/client.go +++ b/agentplane/sandboxes/environmentruntimeclient/client.go @@ -15,7 +15,7 @@ import ( ) // Config describes one call through the Runner Gateway proxy. Every call owns a fresh connection so -// DNS can resolve a new connection holder after a handoff. +// the stable Service can select a ready Gateway replica after a rollout. type Config struct { TLS *peertls.ClientTLS Target string @@ -28,7 +28,7 @@ type Config struct { ReplaceOutgoingMetadata bool } -// ExpandTarget fills the Sandbox name and namespace placeholders in a control Service target. +// ExpandTarget fills the Sandbox name and namespace placeholders in a Runner Gateway target. func ExpandTarget(template, sandbox, namespace string) string { return strings.NewReplacer("{sandbox}", sandbox, "{namespace}", namespace).Replace(template) } From f0aa3dd79c2cd6a9fbf3c47e1dd01e53cb10e3ed Mon Sep 17 00:00:00 2001 From: agentydragon-agent <260791330+agentydragon-agent@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:24:25 -0800 Subject: [PATCH 5/5] Describe Service selection in quiesce and route diagnostics --- agentplane/plans/task_dag.md | 5 +++-- agentplane/runner_gateway/settings.py | 2 +- agentplane/sandbox_controller/README.md | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/agentplane/plans/task_dag.md b/agentplane/plans/task_dag.md index 926551a5257..a01e67b28c7 100644 --- a/agentplane/plans/task_dag.md +++ b/agentplane/plans/task_dag.md @@ -2452,10 +2452,11 @@ the same `UNAVAILABLE` symptom by pausing after the first ready Lease update, th publication-before-local-installation window by installing the fenced connection first. This proves that ordering could cause the symptom, but does not prove it caused the original same-name-replacement failure. Readiness remains a recent observation and cannot guarantee a later call after connection -loss. The Gateway error now reports local connection availability unless Lease ownership is known. +loss. The Gateway error now reports local connection availability; connection-Lease ownership no longer +participates in Runtime routing. If the system-test failure recurs, capture the Sandbox UID, runner incarnation and boot ID, Gateway -replica, Lease epoch and holder, EndpointSlice route, and local connection state at the failed poll. +replica, selected Service endpoint, and local observer/connection state at the failed poll. PR #14851 adds a bounded retry for transient `UNAVAILABLE` in the test's read-only `ListSessions` poll. This handles the test poll only; it does not establish that the publication race caused the original same-name-replacement failure or prove the underlying failure is fixed. Do not retry diff --git a/agentplane/runner_gateway/settings.py b/agentplane/runner_gateway/settings.py index 4234c9fc30c..f305d899959 100644 --- a/agentplane/runner_gateway/settings.py +++ b/agentplane/runner_gateway/settings.py @@ -71,7 +71,7 @@ class Settings(BaseModel): rpc: Listener = Field(description="The shared TLS gRPC listener and its explicitly enabled auth modes.") gateway_tls: ClientTLS = Field( - description="Verify the Session Dispatcher SVID on this Gateway's named control-Service calls." + description="Verify the Session Dispatcher SVID on this Gateway's stable-Service calls." ) proxy_port: int = Field( default=8080, diff --git a/agentplane/sandbox_controller/README.md b/agentplane/sandbox_controller/README.md index f3ae2dea470..b58e5b22f88 100644 --- a/agentplane/sandbox_controller/README.md +++ b/agentplane/sandbox_controller/README.md @@ -97,8 +97,8 @@ on `:8081`, and replicas elect a leader by a Lease in the namespace. garbage collection removes afterwards, never goes while a container can still write to it. - **The runner seals, the controller asks.** The seal is a unary `Runner.Quiesce` through the gateway proxy. The controller records the last request time for that runner incarnation before calling, - and makes at most three attempts on transient `Unavailable` replies within one 30-second handshake, resolving the elected - Gateway again on each try and fencing every call to that same incarnation Pod UID. A changed + and makes at most three attempts on transient `Unavailable` replies within one 30-second handshake, opening a new connection through the stable + Gateway Service on each try and fencing every call to that same incarnation Pod UID. A changed incarnation or refused caller ends the handshake. A failed or unfinished handshake then waits up to one minute before another attempt. Runner stage changes enqueue it sooner; `quiesced` completes without another call. The recorded deadline survives