From ccd51a897b2bb3a220a0ec2f4e0a7ca5eb54e86e Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 15:37:36 +0100 Subject: [PATCH 01/10] feat(platform): add the CE-21 agent host-map writer mcp/ops/agent_hosts.py resolves the Services agents call to their final ClusterIPs (following in-cluster ExternalName targets, skipping outside and headless ones) and writes one ConfigMap key per hostname, short and FQDN. It retries until the required gateway names resolve and talks to the API server with the Job's service-account token (no kubectl image). Co-Authored-By: Claude Opus 5.5 --- mcp/ops/__init__.py | 0 mcp/ops/agent_hosts.py | 164 +++++++++++++++++++++++++++++++ tests/test_agent_hosts_writer.py | 94 ++++++++++++++++++ 3 files changed, 258 insertions(+) create mode 100644 mcp/ops/__init__.py create mode 100644 mcp/ops/agent_hosts.py create mode 100644 tests/test_agent_hosts_writer.py diff --git a/mcp/ops/__init__.py b/mcp/ops/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mcp/ops/agent_hosts.py b/mcp/ops/agent_hosts.py new file mode 100644 index 0000000..417e531 --- /dev/null +++ b/mcp/ops/agent_hosts.py @@ -0,0 +1,164 @@ +"""CE-21: write the agent host map — one ConfigMap key per hostname, value = its ClusterIP. + +Agent pods run without DNS (`dnsPolicy: None`). They reach the few in-cluster names they call +through `hostAliases`, filled from this map by the MutatingAdmissionPolicy (K8s >= 1.36) or by +`zelkor deploy`. ExternalName Services only work through DNS, so each one is followed to the +ClusterIP of its target. Envoy routes by Host, so every gateway name maps to the same IP. + +Runs as a plain Job (not a Helm hook) in the zelkor-mcp image and talks to the API server with +the Job's service-account token. No kubectl. +""" +from __future__ import annotations + +import json +import logging +import os +import ssl +import sys +import time +import urllib.error +import urllib.request +from typing import Callable, Optional + +logger = logging.getLogger("zelkor-agent-hosts") + +SA_DIR = "/var/run/secrets/kubernetes.io/serviceaccount" +CLUSTER_SUFFIX = ".svc.cluster.local" + +# get(path) -> dict or None (404); put(path, body) -> None +Getter = Callable[[str], Optional[dict]] +Putter = Callable[[str, dict, str], None] + + +def _split_external(name: str, default_ns: str) -> tuple[str, str]: + """`svc.ns.svc.cluster.local` / `svc.ns` / `svc` -> (svc, ns).""" + host = name.rstrip(".") + if host.endswith(CLUSTER_SUFFIX): + host = host[: -len(CLUSTER_SUFFIX)] + elif host.endswith(".svc"): + host = host[: -len(".svc")] + parts = host.split(".") + return parts[0], (parts[1] if len(parts) > 1 else default_ns) + + +def resolve_cluster_ip(get: Getter, name: str, namespace: str, *, hops: int = 3) -> Optional[str]: + """ClusterIP behind a Service, following ExternalName targets inside the cluster.""" + for _ in range(hops + 1): + svc = get(f"/api/v1/namespaces/{namespace}/services/{name}") + if not svc: + return None + spec = svc.get("spec") or {} + if spec.get("type") == "ExternalName": + target = str(spec.get("externalName") or "") + if not target.rstrip(".").endswith(CLUSTER_SUFFIX): + return None # outside the cluster: agents must not reach it directly + name, namespace = _split_external(target, namespace) + continue + ip = str(spec.get("clusterIP") or "") + return ip if ip and ip != "None" else None + return None + + +def build_map(get: Getter, services: list[str], namespace: str) -> dict[str, str]: + """{short name and FQDN: ClusterIP} for every listed Service that resolves.""" + out: dict[str, str] = {} + for svc in services: + ip = resolve_cluster_ip(get, svc, namespace) + if not ip: + continue + out[svc] = ip + out[f"{svc}.{namespace}{CLUSTER_SUFFIX}"] = ip + return out + + +def write_configmap(get: Getter, put: Putter, name: str, namespace: str, data: dict[str, str], + release: str) -> None: + path = f"/api/v1/namespaces/{namespace}/configmaps" + body = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": name, + "namespace": namespace, + "labels": { + "app.kubernetes.io/instance": release, + "app.kubernetes.io/component": "agent-hosts", + "zelkor.io/intent": "security.dnsLockdown", + }, + }, + "data": data, + } + if get(f"{path}/{name}") is None: + put(path, body, "POST") + else: + put(f"{path}/{name}", body, "PUT") + + +def run(get: Getter, put: Putter, *, services: list[str], required: list[str], namespace: str, + configmap: str, release: str, deadline_s: float, interval_s: float = 5.0, + sleep: Callable[[float], None] = time.sleep, clock: Callable[[], float] = time.monotonic) -> int: + start = clock() + while True: + data = build_map(get, services, namespace) + missing = [s for s in required if s not in data] + if not missing: + write_configmap(get, put, configmap, namespace, data, release) + logger.info("agent host map written", extra={ + "event": "agent_hosts_written", "configmap": configmap, "hostnames": len(data)}) + return 0 + if clock() - start >= deadline_s: + logger.error("required services never resolved", extra={ + "event": "agent_hosts_failed", "missing": missing}) + return 1 + logger.info("waiting for services", extra={"event": "agent_hosts_wait", "missing": missing}) + sleep(interval_s) + + +def _api_client() -> tuple[Getter, Putter]: + host = os.environ.get("KUBERNETES_SERVICE_HOST", "") + port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") + base = f"https://{host}:{port}" + with open(f"{SA_DIR}/token", encoding="utf-8") as fh: + token = fh.read().strip() + ctx = ssl.create_default_context(cafile=f"{SA_DIR}/ca.crt") + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + + def get(path: str) -> Optional[dict]: + req = urllib.request.Request(base + path, headers=headers) + try: + with urllib.request.urlopen(req, context=ctx, timeout=10) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None + raise + + def put(path: str, body: dict, method: str) -> None: + req = urllib.request.Request(base + path, data=json.dumps(body).encode(), headers=headers, + method=method) + with urllib.request.urlopen(req, context=ctx, timeout=10): + pass + + return get, put + + +def main() -> int: + from zelkor_logging import configure_logging + + configure_logging("zelkor-agent-hosts") + services = os.environ.get("AGENT_HOSTS_SERVICES", "").split() + required = os.environ.get("AGENT_HOSTS_REQUIRED", "").split() + namespace = os.environ.get("AGENT_HOSTS_NAMESPACE", "") + configmap = os.environ.get("AGENT_HOSTS_CONFIGMAP", "") + release = os.environ.get("AGENT_HOSTS_RELEASE", "") + if not (services and namespace and configmap): + logger.error("AGENT_HOSTS_SERVICES, AGENT_HOSTS_NAMESPACE and AGENT_HOSTS_CONFIGMAP are required") + return 2 + get, put = _api_client() + return run(get, put, services=services, required=required, namespace=namespace, + configmap=configmap, release=release, + deadline_s=float(os.environ.get("AGENT_HOSTS_DEADLINE_S", "900"))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_agent_hosts_writer.py b/tests/test_agent_hosts_writer.py new file mode 100644 index 0000000..1cf198e --- /dev/null +++ b/tests/test_agent_hosts_writer.py @@ -0,0 +1,94 @@ +"""CE-21: the agent host-map writer resolves Services (following ExternalName) to ClusterIPs.""" +import os +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "mcp"))) +from ops.agent_hosts import build_map, resolve_cluster_ip, run # noqa: E402 + +NS = "zelkor" +DATAPLANE = "zelkor-zelkor-platform-dataplane.envoy-gateway-system.svc.cluster.local" + + +def _svc(kind="ClusterIP", ip="", external=""): + spec = {"type": kind} + if ip: + spec["clusterIP"] = ip + if external: + spec["externalName"] = external + return {"spec": spec} + + +SERVICES = { + f"/api/v1/namespaces/{NS}/services/zelkor-platform-ai-gateway": _svc("ExternalName", external=DATAPLANE), + f"/api/v1/namespaces/{NS}/services/zelkor-platform-mcp": _svc("ExternalName", external=DATAPLANE), + "/api/v1/namespaces/envoy-gateway-system/services/zelkor-zelkor-platform-dataplane": _svc(ip="10.43.175.236"), + f"/api/v1/namespaces/{NS}/services/zelkor-platform-postgresql": + _svc("ExternalName", external=f"zelkor-platform-postgresql-rw.{NS}.svc.cluster.local"), + f"/api/v1/namespaces/{NS}/services/zelkor-platform-postgresql-rw": _svc(ip="10.43.217.227"), + f"/api/v1/namespaces/{NS}/services/zelkor-platform-langfuse": _svc(ip="10.43.245.107"), + f"/api/v1/namespaces/{NS}/services/zelkor-platform-outside": _svc("ExternalName", external="api.github.com"), + f"/api/v1/namespaces/{NS}/services/zelkor-platform-headless": _svc(ip="None"), +} + + +def _get(objects): + return lambda path: objects.get(path) + + +def test_external_name_is_followed_to_the_target_cluster_ip(): + assert resolve_cluster_ip(_get(SERVICES), "zelkor-platform-ai-gateway", NS) == "10.43.175.236" + assert resolve_cluster_ip(_get(SERVICES), "zelkor-platform-postgresql", NS) == "10.43.217.227" + + +def test_outside_targets_headless_and_missing_services_are_skipped(): + get = _get(SERVICES) + assert resolve_cluster_ip(get, "zelkor-platform-outside", NS) is None + assert resolve_cluster_ip(get, "zelkor-platform-headless", NS) is None + assert resolve_cluster_ip(get, "zelkor-platform-nope", NS) is None + + +def test_map_has_short_and_fqdn_keys_and_gateway_names_share_one_ip(): + data = build_map(_get(SERVICES), ["zelkor-platform-ai-gateway", "zelkor-platform-mcp", + "zelkor-platform-langfuse", "zelkor-platform-nope"], NS) + assert data["zelkor-platform-ai-gateway"] == data["zelkor-platform-mcp"] == "10.43.175.236" + assert data[f"zelkor-platform-mcp.{NS}.svc.cluster.local"] == "10.43.175.236" + assert data["zelkor-platform-langfuse"] == "10.43.245.107" + assert not any(k.startswith("zelkor-platform-nope") for k in data) + + +def test_run_waits_for_required_services_then_writes_once(): + objects = {k: v for k, v in SERVICES.items() if "ai-gateway" not in k} + writes = [] + calls = {"n": 0} + + def sleep(_s): + calls["n"] += 1 + objects[f"/api/v1/namespaces/{NS}/services/zelkor-platform-ai-gateway"] = \ + _svc("ExternalName", external=DATAPLANE) + + rc = run(lambda p: objects.get(p), lambda path, body, method: writes.append((path, method, body)), + services=["zelkor-platform-ai-gateway", "zelkor-platform-mcp"], + required=["zelkor-platform-ai-gateway", "zelkor-platform-mcp"], namespace=NS, + configmap="zelkor-platform-agent-hosts", release="zelkor-platform", deadline_s=60, + sleep=sleep, clock=lambda: 0.0) + assert rc == 0 and calls["n"] == 1 + assert len(writes) == 1 and writes[0][1] == "POST" + assert writes[0][2]["data"]["zelkor-platform-ai-gateway"] == "10.43.175.236" + + +def test_run_fails_when_required_services_never_resolve(): + ticks = iter([0.0, 0.0, 100.0]) + rc = run(lambda p: None, lambda *a: None, services=["zelkor-platform-ai-gateway"], + required=["zelkor-platform-ai-gateway"], namespace=NS, configmap="m", release="r", + deadline_s=50, sleep=lambda _s: None, clock=lambda: next(ticks)) + assert rc == 1 + + +def test_existing_configmap_is_replaced(): + objects = dict(SERVICES) + objects[f"/api/v1/namespaces/{NS}/configmaps/m"] = {"data": {}} + writes = [] + run(lambda p: objects.get(p), lambda path, body, method: writes.append((path, method)), + services=["zelkor-platform-mcp"], required=["zelkor-platform-mcp"], namespace=NS, + configmap="m", release="r", deadline_s=1, sleep=lambda _s: None, clock=lambda: 0.0) + assert writes == [(f"/api/v1/namespaces/{NS}/configmaps/m", "PUT")] From 2e02cf30396520af1337c73abfcfaab43378e817 Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 15:40:59 +0100 Subject: [PATCH 02/10] feat(helm): render the CE-21 host map, admission policy and DNS-free agent egress security.dnsLockdown.enabled (default true) adds a plain per-revision Job (zelkor-mcp image, namespaced RBAC scoped to its ConfigMap) that writes {release}-agent-hosts. On clusters that serve MutatingAdmissionPolicy the chart also renders a per-release policy and binding: new agent pods of this platform (zelkor.io/platform-release) get dnsPolicy None, a dead nameserver and hostAliases from the map; no map denies the pod. The platform agent-egress policy drops port 53. Agents not yet redeployed keep DNS through their own chart's rule (policies are additive). Config matrix: dnsLockdown joins the model and the 3-way isolation set. Co-Authored-By: Claude Opus 5.5 --- .../templates/security/agent-hosts.yaml | 198 ++++++++++++++++++ .../templates/security/networkpolicies.yaml | 2 + charts/zelkor-platform/values.schema.json | 18 ++ charts/zelkor-platform/values.yaml | 5 + tests/config_matrix/matrix.tsv | 125 +++++------ tests/config_matrix/model.pict | 3 +- tests/test_helm_dns_lockdown.py | 95 +++++++++ 7 files changed, 384 insertions(+), 62 deletions(-) create mode 100644 charts/zelkor-platform/templates/security/agent-hosts.yaml create mode 100644 tests/test_helm_dns_lockdown.py diff --git a/charts/zelkor-platform/templates/security/agent-hosts.yaml b/charts/zelkor-platform/templates/security/agent-hosts.yaml new file mode 100644 index 0000000..fb06f52 --- /dev/null +++ b/charts/zelkor-platform/templates/security/agent-hosts.yaml @@ -0,0 +1,198 @@ +{{- include "zelkor-platform.compile" . -}} +{{- /* CE-21: agent pods run without DNS. This writes the host map (one key per hostname, value = + ClusterIP) that fills their hostAliases: the admission policy below on K8s >= 1.36, or + `zelkor deploy` on older clusters. Plain Job, not a hook: Helm must not wait on it. */ -}} +{{- if .Values.security.dnsLockdown.enabled }} +{{- $fullname := include "zelkor-platform.fullname" . }} +{{- $ns := .Release.Namespace }} +{{- $cm := printf "%s-agent-hosts" $fullname }} +{{- $dataplane := include "zelkor-platform.envoyDataplaneHost" . | trim }} +{{- $dpNs := "" }} +{{- if $dataplane }}{{- $dpNs = index (splitList "." $dataplane) 1 }}{{- end }} +{{- $services := list (printf "%s-ai-gateway" $fullname) (printf "%s-mcp" $fullname) (printf "%s-otlp" $fullname) (printf "%s-postgresql" $fullname) (printf "%s-langfuse" $fullname) (printf "%s-valkey" $fullname) }} +{{- range (.Values.__httpApis | default "[]" | fromJsonArray) }} +{{- $services = append $services (printf "%s-api-%s" $fullname .name) }} +{{- end }} +{{- $required := list }} +{{- if and .Values.aiGateway.enabled .Values.aiGateway.inClusterService.enabled $dataplane }} +{{- $required = append $required (printf "%s-ai-gateway" $fullname) }} +{{- end }} +{{- if and (eq (include "zelkor-platform.mcpMcprouteEnabled" .) "true") $dataplane }} +{{- $required = append $required (printf "%s-mcp" $fullname) }} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $fullname }}-agent-hosts + namespace: {{ $ns }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +automountServiceAccountToken: true +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullname }}-agent-hosts + namespace: {{ $ns }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +rules: + - apiGroups: [""] + resources: [services] + verbs: [get] + - apiGroups: [""] + resources: [configmaps] + verbs: [create] + - apiGroups: [""] + resources: [configmaps] + resourceNames: [{{ $cm | quote }}] + verbs: [get, update] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullname }}-agent-hosts + namespace: {{ $ns }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $fullname }}-agent-hosts +subjects: + - kind: ServiceAccount + name: {{ $fullname }}-agent-hosts + namespace: {{ $ns }} +{{- if and $dpNs (ne $dpNs $ns) }} +--- +# Read the Envoy dataplane Service that the gateway ExternalNames point at. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullname }}-agent-hosts + namespace: {{ $dpNs }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +rules: + - apiGroups: [""] + resources: [services] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullname }}-agent-hosts + namespace: {{ $dpNs }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $fullname }}-agent-hosts +subjects: + - kind: ServiceAccount + name: {{ $fullname }}-agent-hosts + namespace: {{ $ns }} +{{- end }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + # Per revision: a Job's pod template is immutable, so a fixed name would break `helm upgrade`. + name: {{ $fullname }}-agent-hosts-{{ .Release.Revision }} + namespace: {{ $ns }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} + app.kubernetes.io/component: agent-hosts +spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 10 + activeDeadlineSeconds: {{ int .Values.security.dnsLockdown.writerDeadlineSeconds }} + template: + metadata: + labels: + {{- include "zelkor-platform.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: agent-hosts + spec: + restartPolicy: OnFailure + serviceAccountName: {{ $fullname }}-agent-hosts + {{- include "zelkor-platform.imagePullSecrets" . | nindent 6 }} + containers: + - name: writer + image: {{ include "zelkor-platform.image" .Values.mcp.image | quote }} + imagePullPolicy: {{ include "zelkor-platform.imagePullPolicy" (dict "root" . "image" .Values.mcp.image) }} + command: ["python", "/app/mcp/ops/agent_hosts.py"] + env: + - name: AGENT_HOSTS_SERVICES + value: {{ join " " $services | quote }} + - name: AGENT_HOSTS_REQUIRED + value: {{ join " " $required | quote }} + - name: AGENT_HOSTS_NAMESPACE + value: {{ $ns | quote }} + - name: AGENT_HOSTS_CONFIGMAP + value: {{ $cm | quote }} + - name: AGENT_HOSTS_RELEASE + value: {{ .Release.Name | quote }} + - name: AGENT_HOSTS_DEADLINE_S + value: {{ int .Values.security.dnsLockdown.writerDeadlineSeconds | quote }} + {{- include "zelkor-platform.logEnv" (dict "root" . "name" "zelkor-agent-hosts") | nindent 12 }} + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {cpu: 200m, memory: 128Mi} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/MutatingAdmissionPolicy" }} +--- +# K8s >= 1.36: every new agent pod of this platform gets no DNS and the current host map. +# One JSONPatch per entry: a CEL-built list of objects panics the 1.36.4 API server. +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingAdmissionPolicy +metadata: + name: {{ printf "%s-%s-agent-no-dns" $fullname $ns | trunc 253 }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +spec: + paramKind: + apiVersion: v1 + kind: ConfigMap + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE] + resources: [pods] + objectSelector: + matchLabels: + zelkor.io/platform-release: {{ .Release.Name | quote }} + matchExpressions: + - key: zelkor.io/workload-type + operator: In + values: [agent, app] + failurePolicy: Fail + reinvocationPolicy: IfNeeded + mutations: + - patchType: JSONPatch + jsonPatch: + expression: > + [ + JSONPatch{op: "add", path: "/spec/dnsPolicy", value: "None"}, + JSONPatch{op: "add", path: "/spec/dnsConfig", + value: Object.spec.dnsConfig{nameservers: ["127.0.0.1"]}}, + JSONPatch{op: "add", path: "/spec/hostAliases", value: []} + ] + params.data.map(h, JSONPatch{op: "add", path: "/spec/hostAliases/-", + value: Object.spec.hostAliases{ip: params.data[h], hostnames: [h]}}) +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingAdmissionPolicyBinding +metadata: + name: {{ printf "%s-%s-agent-no-dns" $fullname $ns | trunc 253 }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +spec: + policyName: {{ printf "%s-%s-agent-no-dns" $fullname $ns | trunc 253 }} + paramRef: + name: {{ $cm }} + namespace: {{ $ns }} + # No map yet: refuse the pod rather than start it with DNS. Its ReplicaSet retries. + parameterNotFoundAction: Deny +{{- end }} +{{- end }} diff --git a/charts/zelkor-platform/templates/security/networkpolicies.yaml b/charts/zelkor-platform/templates/security/networkpolicies.yaml index d5e4256..cfcff76 100644 --- a/charts/zelkor-platform/templates/security/networkpolicies.yaml +++ b/charts/zelkor-platform/templates/security/networkpolicies.yaml @@ -18,6 +18,7 @@ spec: policyTypes: - Egress egress: + {{- if not .Values.security.dnsLockdown.enabled }} - to: - namespaceSelector: matchLabels: @@ -27,6 +28,7 @@ spec: port: 53 - protocol: TCP port: 53 + {{- end }} - to: - podSelector: matchLabels: diff --git a/charts/zelkor-platform/values.schema.json b/charts/zelkor-platform/values.schema.json index f9e7ec9..63538da 100644 --- a/charts/zelkor-platform/values.schema.json +++ b/charts/zelkor-platform/values.schema.json @@ -4142,6 +4142,24 @@ }, "additionalProperties": false }, + "dnsLockdown": { + "type": "object", + "description": "Values under security.dnsLockdown.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Value for security.dnsLockdown.enabled." + }, + "writerDeadlineSeconds": { + "type": [ + "integer", + "string" + ], + "description": "Value for security.dnsLockdown.writerDeadlineSeconds." + } + }, + "additionalProperties": false + }, "mTLS": { "type": "object", "description": "Values under security.mTLS.", diff --git a/charts/zelkor-platform/values.yaml b/charts/zelkor-platform/values.yaml index afe14a4..92d3d72 100644 --- a/charts/zelkor-platform/values.yaml +++ b/charts/zelkor-platform/values.yaml @@ -711,6 +711,11 @@ security: suspiciousOnProbePlusError: true networkPolicies: enabled: false + # CE-21: agent pods get no DNS (dnsPolicy None + hostAliases from the host map). The platform + # writes the map; agents get it from the admission policy (K8s >= 1.36) or `zelkor deploy`. + dnsLockdown: + enabled: true + writerDeadlineSeconds: 900 mTLS: enabled: false # Opt-in admission policy (needs Kyverno >= 1.12 in the cluster): Pods may only run Zelkor images diff --git a/tests/config_matrix/matrix.tsv b/tests/config_matrix/matrix.tsv index 254e423..cf4d614 100644 --- a/tests/config_matrix/matrix.tsv +++ b/tests/config_matrix/matrix.tsv @@ -1,61 +1,64 @@ -@gateway profile perimeter.guardrails perimeter.sandbox perimeter.dataTools.postgres perimeter.dataTools.qdrant databases.mode postgresql.enabled valkey.enabled clickhouse.enabled qdrant.enabled seaweedfs.enabled highAvailability.enabled gateway.tls.enabled security.networkPolicies.enabled security.imageVerification.enabled security.sandbox.enabled security.sandbox.provisioning.mode security.sandbox.executionLog.enabled platform.telemetry.format platform.telemetry.nemoOtel.enabled platform.telemetry.langfuse.enabled platform.telemetry.langfuse.worker.enabled platform.telemetry.langfuse.publicHttpRoute.enabled observability.serviceMonitor.enabled workspace.models.enabled workspace.models.rateLimit.enabled workspace.policies.nemo.enabled workspace.policies.nemo.intercept.enabled workspace.policies.nemo.selfCheck.enabled workspace.tools.enabled workspace.tools.postgresMCP.enabled workspace.tools.qdrantMCP.enabled workspace.tools.aigatewayMCP.enabled workspace.tools.sandboxMCP.enabled workspace.tools.objectMCP.enabled workspace.tools.policy.rateLimit.enabled workload.agents.enabled workload.agents.redisBroker.enabled workload.agents.graphModules.enabled workload.agents.mcpInject.enabled -greenfield perimeter false false false false external true true true false false true false true false true auto false text true true false true false true true false true true false true true true true true false true false true true -layered perimeter true true true true external false false false true true false true true true false daemonset true json false false true false true false false true false false false false false false false false true true true false false -shared perimeter true false true false in-cluster-basic true false false false false false true false false false auto true text false true true false false false true true true false true true true false true false true false false false true -greenfield perimeter false false false true external false true true true true true false false true false daemonset false json true false false true true true false false false true true false false true false true false true true true false -greenfield perimeter false true true true operator-cr true true false true false true true false true true auto false json false true false false true true true true false true false true false false true true true true true true true -layered perimeter false true false false external false false true false true false false true false false daemonset true text true false true true false false false false true false true false true true false false false false false false false -layered perimeter true true false false in-cluster-basic false true false true true false false true false true auto false text false true true true true true false true false false true true true true false true false true true true false -layered perimeter true false true true in-cluster-basic true false true false false true true false true true daemonset true json true false false false true false true false true true false false false true false true true false false false true -layered full false false false false operator-cr true false true true false true true true true false daemonset false text true true true true false false false false true true false false true true true false false true false true false -shared perimeter true false false true operator-cr false true true false true false false true false true auto true json false false false true false true false false true false true true false false false false true true true false true -greenfield perimeter true false false false in-cluster-basic false true false false true false false false false true auto true json true false true false false false true false false false false false true false false false false true false false false -layered perimeter true true true true external false false true false true true false false false false daemonset false text true false false false true true true true false false true true false false true true false true true false true -layered perimeter false true true false external false true false true true false false false false true auto true text false true true true false true false true false false false true true false true false false false true true false -greenfield perimeter true true true false operator-cr false false false true true false true true true true daemonset true json false false true false true false false true false false false false false false false false true true true false false -shared perimeter false true false true operator-cr false true true true true false false false true false daemonset false json true false false true true true false false false true true false false true false true false false true true false -greenfield perimeter true true false true operator-cr true true false false true false false false false true auto true json false false false false true true true true false false true true true false false true true false true false true -layered perimeter false false true true operator-cr false false true false false true true true true true daemonset true json true false false false false false true false true true true false false false true false true true false false true -layered perimeter false false true true in-cluster-basic true true false true false true true true true false auto false json false true false false true true true true false true true true false false true true true false true true true -shared perimeter true true false true external false true false false true true false false false false auto true json false false false false true true true true false false true true false false false true true false true false true -greenfield full false false false false operator-cr false true false false true false false true false true auto true json false false false false true true true true false false true true false false false true true false true false true -layered perimeter false false true false in-cluster-basic true true false false true true false true false false auto false text false false false true false true true true true false false true true false true true true true true false true -layered perimeter true true false false operator-cr false true false true true true true false false true daemonset false text false true false true true false true false false true false false false true true true true false true false true -shared perimeter true true false true operator-cr false true false true false false false true true false auto false json true false false true false false true true true true true true true true true false false true false false false -greenfield perimeter true true true true external true false true true false true true true true true daemonset false text true true true true false false false false true true true false true true true false false false false true false -greenfield perimeter false true false true in-cluster-basic true true false true true true false true false true auto true text true false true false true false true false false true false true true true true false false true false true false -greenfield perimeter false true false true external true false true true true false false true true false auto true text true true false false true false false true true false false false true false true false true false true false false -shared perimeter false false true false external true false true false true true true true false false daemonset true text false true true false true true false false false false true false false false false true true true true true true -layered perimeter false true true false external true false false true true true false true false true daemonset true json false true false true true false true true true true true false true true true true false true false false false -layered full false false false false in-cluster-basic true true false true false false true false true false auto true json false true true true false false false false true true true false true true false false true true false true false -shared perimeter true false true true operator-cr false false false false false false true true false true auto true text true false false true false true true true true true false true false true false true false false false false true -greenfield perimeter true true true false operator-cr true false false false false false true false false false auto true text false true true false false false true true true false true true true false true false true true false false true -shared perimeter false false true true in-cluster-basic true false true false false false false false true false auto true text true false true false true false true true false true false false false false false true true true false false true -greenfield perimeter true false true false operator-cr false true false false false true true true false false daemonset false json false false true true false true true false false true false true false true false true false false false true true -layered perimeter true false true true operator-cr true false false true false false true false false false auto true text true true false true true true false false false false true false true true true false false true false true false -shared perimeter true true false false operator-cr false true false true true true true false false false daemonset false json false true false true false true false false true false false true true true true false false true true true false -shared perimeter false true false false external true true true true true true false true true true daemonset false json false true true false true false false false false false false false true false true false true false true true false -greenfield perimeter false true true true external false false true false false true true false false true daemonset false text true true true true false true false false true true false true false true true false true true false false true -greenfield perimeter true false true false in-cluster-basic false true false true false true false true true false daemonset true text false true false false true true true false true true false true true true true false true false true false false -shared full false false false false external true false true false true true true false true false daemonset false text true true false false true true false false true true true true true true true true true false false true true -shared perimeter false true true false external true false false true false true true true true true daemonset false text true true true false true false true true false true true false true true true true false true false true false -greenfield full false false false false external false true true true true true false false false true auto true text false true false false true true true false true true false true false true false true true true false true false -layered perimeter true true false true operator-cr true true true false true true false true false false auto false json true true true false false false false false false false true false true false true false false false false false false -greenfield perimeter false true true false in-cluster-basic false false true false false true true false true false daemonset true json false true true true false false true true true false true true true true false false true true false true false -shared full false false false false in-cluster-basic false true false false true false true true true true daemonset false json false true false false false true true true false false true true true false true true false true false false false -shared perimeter true false false true external false false true true true true true false true true daemonset true json false false false true true true false false false true true false false true true true false false true false true -greenfield full false false false false operator-cr false true true true false true true true true true daemonset true json true true true false true false false false false true false true false false false true true true false true true -greenfield full false false false false in-cluster-basic false false true true false true true false true true daemonset false text true true true true false false false false true true true false false true true false false false false true false -shared perimeter false true true false operator-cr true true false false false false false false false true auto false text true true true false false false true true true false true true true false false false true false false true false -layered full false false false false external true true true true true true true true true false daemonset true json true false true false false true false false true true true false true false false false false false false false false -greenfield perimeter false true true false external true true true false false true false false true true auto true text false true true true false false true true false true true true false false true false true false false false false -shared full false false false false external false true true true true false false false true false auto true text false true true false false true false false true false false true false true true true false true true true false -layered full false false false false operator-cr false true true true false true false false true true daemonset true text false false false true true false true false true true false true true false false true false false false false false -shared perimeter false true false true in-cluster-basic false false true true false true true true false false auto true json false true false true false false false true true false true true true false true true false true true true true -layered full false false false false in-cluster-basic true false true false false true false true true true daemonset false json true false true false true false false false false false true true false true true false false true false false true -shared full false false false false external false true true true false false false true false false auto true text true false false false false true true false true true true false false true false false false true true true true -shared perimeter true true false true in-cluster-basic false false false true true true false true true true daemonset true text true false true false false false false true true true false true false true true false false false false false false -greenfield full false false false false external true true false true false true false false true false auto true json true true true true true false false true false false true false false true true true false true true true false -shared perimeter true false false true in-cluster-basic true true false false false false false false true true daemonset false json true false false true true false false true true false true false true false false true false false false true true -shared full false false false false operator-cr true true false false false false true false false true auto true text false false true false true false false true false true true true true true false false true false false false false -greenfield perimeter true true true true in-cluster-basic true true true true false true true true false true auto true text false true false false false true false false false true true true false true true false true false true true false +@gateway profile perimeter.guardrails perimeter.sandbox perimeter.dataTools.postgres perimeter.dataTools.qdrant databases.mode postgresql.enabled valkey.enabled clickhouse.enabled qdrant.enabled seaweedfs.enabled highAvailability.enabled gateway.tls.enabled security.networkPolicies.enabled security.dnsLockdown.enabled security.imageVerification.enabled security.sandbox.enabled security.sandbox.provisioning.mode security.sandbox.executionLog.enabled platform.telemetry.format platform.telemetry.nemoOtel.enabled platform.telemetry.langfuse.enabled platform.telemetry.langfuse.worker.enabled platform.telemetry.langfuse.publicHttpRoute.enabled observability.serviceMonitor.enabled workspace.models.enabled workspace.models.rateLimit.enabled workspace.policies.nemo.enabled workspace.policies.nemo.intercept.enabled workspace.policies.nemo.selfCheck.enabled workspace.tools.enabled workspace.tools.postgresMCP.enabled workspace.tools.qdrantMCP.enabled workspace.tools.aigatewayMCP.enabled workspace.tools.sandboxMCP.enabled workspace.tools.objectMCP.enabled workspace.tools.policy.rateLimit.enabled workload.agents.enabled workload.agents.redisBroker.enabled workload.agents.graphModules.enabled workload.agents.mcpInject.enabled +shared perimeter false false false false in-cluster-basic true false false true false true true false false false false auto false text false true true true false true true false false true true true true false false true false false true false false +greenfield perimeter false true true true operator-cr false true true false true false false true true true true daemonset true json true false false false true false false true true false true false false true true false true false false true true +greenfield perimeter true true false false in-cluster-basic false true false false false true false false true true false auto true text true true true false false false false false false false true false true false true false false false false false true +layered perimeter true false true true external true false true true true false true false false false true daemonset false json false false false true true true true true true true true true false true true true true false true true false +layered perimeter true false true false in-cluster-basic true true false true true false false true true false false auto true json true true false true false true false false true false true true true true false false true true true true true +greenfield perimeter false true false true external false false true false false true true true true true true daemonset false text false false true false true false false true false true true false false false false true false false false false false +greenfield perimeter true true true false operator-cr true true true false false false true false false false true daemonset true text true true true true true false true false true true false false true false false false true false true false false +greenfield perimeter false false false true operator-cr false false false true true true false false false true false auto false text false false false false false true true false false false true true false true false true false true false true true +layered perimeter false true false false external false false false true false true true true false false false auto true json false true true false false true true true true false false false true true true false false false true false true +shared full false false false false external true true true false true false false false false true true auto false json true false true true true false false false false false true true true false false true true true false true false +shared perimeter true false true false operator-cr true true false false false false false false false true true daemonset false text false true false false false true true true true true false false false false true false false false true false true +layered perimeter true true true true in-cluster-basic false false true true false true false true false false false daemonset true text true false false true true false false true false true false true false false false true true false false true false +greenfield perimeter true true false true external false false true true true true true true true false false daemonset true text false true false false false true true true true true false false false true true false false true true false true +greenfield perimeter false false true true operator-cr true false false true true false true false true false false daemonset false json false false false true true true true true true true true true false true false true true false true true false +shared perimeter true false false true external false false false true false true true true false false false daemonset true text false true false false false true true true true true false false true true true false false true true false true +greenfield perimeter false false true false operator-cr true true false false false false false true true true false auto false json true false true true true false false false false false false true true false false true true true false true false +layered perimeter true false false false operator-cr true true false false true false true true false true false daemonset false text false true false false false true true false true true false false false true true false false false true false true +shared perimeter true true true true external false true true false true false false false false true false daemonset true json true false false false true false false true true false true false false true true false true false false true true +layered perimeter false true false true external false false true false false true true true true true false daemonset false text false false true false true false true true false true true false false false true true false true false false false +greenfield perimeter false true true false external false false false true false true true true false false true daemonset true text false true false false false true true true true true false false false true true false false true true false true +greenfield perimeter false false false true in-cluster-basic false false false true true true false false false true true auto false json false false false false false true false true false false false true false true true true false false false true true +layered perimeter true true false true operator-cr false false false true false true true false true false false daemonset true text false true false false false true true true true true false false false true true false false true true false true +layered perimeter false false true true external false false false true false true true false true false true daemonset true text false true false false false true true true true true false false false true true false false false true false true +shared perimeter true true true true in-cluster-basic false false false true false true true false false false true daemonset true text false true false false false true true true true true true false false true true false false true true false true +greenfield perimeter true true true false in-cluster-basic true true true false false false true false false false false daemonset true json true true true true true false false true true true true false true false true false true true true false false +layered perimeter true true false true external false false false true false true true true false false true daemonset true text false true false false false true true true true true true false false true true false false true true false true +shared perimeter false true true false in-cluster-basic false false false true false true true true true false false daemonset true text false true false false false true true true true true true false false true true false false false true false true +shared perimeter true false true true in-cluster-basic false false false true false true true false true false false daemonset true text false true false false false true true true true true true false false true true false false false true false true +layered perimeter true true false false operator-cr true true true false true false false false true true true auto false json true false true true true false false false false false false true true false false true true false false true false +layered perimeter false false true false operator-cr true true true false true false false true false true true auto false json true false true true true false false false false false true true true false false true true true false true false +layered perimeter false true false false in-cluster-basic false true false false false true false false false true true auto true text true true true false false false false false false false true false true false false false false false false false true +shared perimeter false true true false operator-cr true true true false true false false true false true false auto false json true false true true true false false false false false false true false false false true true true false true false +layered perimeter false false true true in-cluster-basic false false false true false true true true false false true daemonset true text false true false false false true true true true true true false false true true false false true true false true +shared perimeter true true false true operator-cr false false false true false true true false false false false daemonset true text false true false false false true true true true true true false false true true false false false true false true +greenfield perimeter true false false false in-cluster-basic false false false true false true true true true false false daemonset true text false true false false false true true true true true false false false true true false false true true false true +shared perimeter true false false true external true true true false true false false true true true false auto false json true false true true true false false false false false true true true false false true true false false true false +shared perimeter false true true false external false false false true false true true true true false true daemonset true text false true false false false true true true true true false false false true true false false true true false true +shared perimeter false true false true in-cluster-basic false false true true true true true false false false true auto true json true false true true true false false false false false false true true true false true true false false true false +greenfield perimeter true false false true operator-cr true true true false true false false true false true true auto false json true false true true true false false false false false false true true false false true true true false true false +shared perimeter true false false true operator-cr true true true false true false false true true true true auto false json true false true true true false false false false false false true true false false true true true false true false +shared full false false false false operator-cr false false false true false true true true true false false daemonset true text false true false false false true true true true true true false false true true false false false true false true +layered full false false false false in-cluster-basic false true true true true true true false true false true daemonset true json false false false false false false false false true true false false true true true true true false true false true +greenfield perimeter false false true true in-cluster-basic false false true false true false true true false false true auto true json false true true true false false false true false false false false false true true false false true true true true +layered full false false false false in-cluster-basic true false true false true false false true false true true daemonset false text true true false false false false false false true false true true false true true false false true false false true +greenfield full false false false false external true true false false false true false false false true true auto true text false false true false false false true true true false false true true true true true false false false false false +greenfield full false false false false external false true true false false true true false false false false daemonset false json true false true true false true true false false false true true false false false true true true true true true +greenfield full false false false false in-cluster-basic true true false true true false false true false true true auto true text false true true false false false true false false false false true true true true true true true true true true +shared perimeter false true false false external true true true false true false false false true true false auto false json true false true true true false false false false false true true true false false true true false false true false +layered full false false false false operator-cr true false true false false false true true true false false auto true text true false true false true true true true true true true true true false true true true true true true false +shared full false false false false external true false true false false true true false false true true auto true text false true true true true false true false false true false false true true true true true false false false true +layered perimeter false false true false in-cluster-basic true true true false true false false false true true false auto false json true false true true true false false false false false false true true false false true true true false true false +shared full false false false false in-cluster-basic false true false true true true false true true false true daemonset false text true true false true false false true true true false false false true true true false false true true false false +greenfield full false false false false in-cluster-basic true true false true false false true true true false true auto true json false true true true false true true false false false true true true false true true true false true true false +shared full false false false false external true true true true false false false false false true false daemonset false json true false false false true true false false true false true false false true false false false false false true true +greenfield full false false false false external false false true true false true false false true true false auto true json false false false false true true false false true true true false false true false false false false false true false +greenfield full false false false false in-cluster-basic false false false false false false true true true false false daemonset true json false false false true false true true false false true false true false false false true false true false true true +shared perimeter false false true false operator-cr false false false true true true false false false false true daemonset false json true true false true true false true false true false true true true true false false true true false false true +shared perimeter true true false true operator-cr true true false false false true false false true true false auto true text false true false false false false true true false true true false false true true false false false false false true +layered full false false false false operator-cr false false false false false false true false false false true auto false text true true true true true true false false true true true true false true true true true false false false true +shared full false false false false in-cluster-basic false false false true false true false true false true false auto false json false false true true false true false true false true false true false false false false false true true true false +layered full false false false false external false false false true true false true false true true false daemonset true json true false true true false false false true true false false false false true false false true true true true false +greenfield perimeter true true true false operator-cr true true true true false true true true true false true daemonset false text true false false false true true false false true true true true true false false true true false false true true +shared full false false false false operator-cr true false false true true true true false true false false auto false text false false true false true false false true false false true false false false false false false false true true false diff --git a/tests/config_matrix/model.pict b/tests/config_matrix/model.pict index d4d3ec1..70bf261 100644 --- a/tests/config_matrix/model.pict +++ b/tests/config_matrix/model.pict @@ -18,6 +18,7 @@ seaweedfs.enabled: true, false highAvailability.enabled: false, true gateway.tls.enabled: false, true security.networkPolicies.enabled: false, true +security.dnsLockdown.enabled: true, false security.imageVerification.enabled: false, true security.sandbox.enabled: true, false security.sandbox.provisioning.mode: auto, daemonset @@ -46,7 +47,7 @@ workload.agents.graphModules.enabled: false, true workload.agents.mcpInject.enabled: false, true # Isolation promises interact: cover every 3-way combination of these. -{ @gateway, security.networkPolicies.enabled, security.sandbox.enabled, workspace.tools.enabled, workload.agents.enabled } @ 3 +{ @gateway, security.networkPolicies.enabled, security.dnsLockdown.enabled, security.sandbox.enabled, workspace.tools.enabled, workload.agents.enabled } @ 3 # templates/security/networkpolicies.yaml fails: NetworkPolicies need agents or tools to enforce anything # (full profile; perimeter turns them on itself and wrapped agents bring their own). diff --git a/tests/test_helm_dns_lockdown.py b/tests/test_helm_dns_lockdown.py new file mode 100644 index 0000000..3827116 --- /dev/null +++ b/tests/test_helm_dns_lockdown.py @@ -0,0 +1,95 @@ +"""CE-21 render contract: agent pods without DNS (platform chart side).""" +from __future__ import annotations + +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +CHART = ROOT / "charts" / "zelkor-platform" +LOCAL = ROOT / "profiles" / "values-local.yaml" +MAP_API = "admissionregistration.k8s.io/v1/MutatingAdmissionPolicy" + + +def _render(*extra: str, check: bool = True) -> subprocess.CompletedProcess[str]: + cmd = ["helm", "template", "zelkor-platform", str(CHART), "--namespace", "zelkor", + "-f", str(LOCAL), "--set", "platform.tenants.jwt.remoteJwksUri=", *extra] + proc = subprocess.run(cmd, check=False, capture_output=True, text=True) + if check: + assert proc.returncode == 0, proc.stderr + return proc + + +def _docs(*extra: str) -> list[dict]: + return [d for d in yaml.safe_load_all(_render(*extra).stdout) if d] + + +def _kind(docs: list[dict], kind: str) -> list[dict]: + return [d for d in docs if d.get("kind") == kind] + + +def _writer_env(docs: list[dict]) -> dict[str, str]: + jobs = [j for j in _kind(docs, "Job") if j["metadata"]["name"].startswith("zelkor-platform-agent-hosts-")] + assert len(jobs) == 1, [j["metadata"]["name"] for j in _kind(docs, "Job")] + return {e["name"]: e.get("value", "") for e in jobs[0]["spec"]["template"]["spec"]["containers"][0]["env"]} + + +def _agent_egress(docs: list[dict]) -> dict: + return next(d for d in _kind(docs, "NetworkPolicy") if d["metadata"]["name"] == "zelkor-platform-agent-egress") + + +def _allows_dns(policy: dict) -> bool: + return any(p.get("port") == 53 for rule in policy["spec"].get("egress", []) for p in rule.get("ports", [])) + + +def test_default_writes_the_host_map_and_agent_egress_has_no_dns(): + docs = _docs() + env = _writer_env(docs) + services = env["AGENT_HOSTS_SERVICES"].split() + assert "zelkor-platform-ai-gateway" in services and "zelkor-platform-mcp" in services + assert env["AGENT_HOSTS_REQUIRED"].split() == ["zelkor-platform-ai-gateway", "zelkor-platform-mcp"] + assert env["AGENT_HOSTS_CONFIGMAP"] == "zelkor-platform-agent-hosts" + job = next(j for j in _kind(docs, "Job") if j["metadata"]["name"].startswith("zelkor-platform-agent-hosts-")) + # A plain Job, never a hook: Helm must not wait on it. + assert "helm.sh/hook" not in (job["metadata"].get("annotations") or {}) + assert not _allows_dns(_agent_egress(docs)) + + +def test_writer_rbac_is_namespaced_and_scoped_to_its_configmap(): + docs = _docs() + assert not _kind(docs, "ClusterRole") or all( + "agent-hosts" not in r["metadata"]["name"] for r in _kind(docs, "ClusterRole")) + roles = [r for r in _kind(docs, "Role") if r["metadata"]["name"] == "zelkor-platform-agent-hosts"] + assert {r["metadata"]["namespace"] for r in roles} == {"zelkor", "envoy-gateway-system"} + own = next(r for r in roles if r["metadata"]["namespace"] == "zelkor") + writes = [rule for rule in own["rules"] if "update" in rule["verbs"]] + assert writes and all(rule.get("resourceNames") == ["zelkor-platform-agent-hosts"] for rule in writes) + + +def test_http_api_routes_join_the_host_map(): + docs = _docs("--set", "workspace.tools.httpApis[0].name=github", + "--set", "workspace.tools.httpApis[0].url=https://api.github.com", + "--set", "workspace.tools.httpApis[0].rules[0].action=Allow") + assert "zelkor-platform-api-github" in _writer_env(docs)["AGENT_HOSTS_SERVICES"].split() + + +def test_admission_policy_only_when_the_api_is_served(): + assert not _kind(_docs(), "MutatingAdmissionPolicy") + docs = _docs("--api-versions", MAP_API) + (policy,) = _kind(docs, "MutatingAdmissionPolicy") + (binding,) = _kind(docs, "MutatingAdmissionPolicyBinding") + selector = policy["spec"]["matchConstraints"]["objectSelector"] + assert selector["matchLabels"] == {"zelkor.io/platform-release": "zelkor-platform"} + assert binding["spec"]["paramRef"] == {"name": "zelkor-platform-agent-hosts", "namespace": "zelkor", + "parameterNotFoundAction": "Deny"} + expr = policy["spec"]["mutations"][0]["jsonPatch"]["expression"] + # One patch per entry; a CEL-built list of objects panics the 1.36.4 API server. + assert '"/spec/hostAliases/-"' in expr and "split(" not in expr + + +def test_lockdown_off_keeps_dns_and_renders_no_writer(): + docs = _docs("--set", "security.dnsLockdown.enabled=false", "--api-versions", MAP_API) + assert not [j for j in _kind(docs, "Job") if "agent-hosts" in j["metadata"]["name"]] + assert not _kind(docs, "MutatingAdmissionPolicy") + assert _allows_dns(_agent_egress(docs)) From 0b7ceb31abddbaeafe7349a002e5895a215ba21f Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 15:45:03 +0100 Subject: [PATCH 03/10] feat(helm,agents): agent pods run without DNS when a host map can fill them charts/zelkor-agent: security.dnsLockdown.enabled (default true) and platform.hostAliases. When zelkor deploy supplies host entries, or the platform admission policy can inject them (K8s >= 1.36 with platform.releaseName), the pod gets dnsPolicy None, nameserver 127.0.0.1, its hostAliases and the labels zelkor.io/dns-lockdown=on and zelkor.io/platform-release; its own egress policy drops port 53. A plain Helm install on an older cluster keeps DNS and is annotated zelkor.io/dns-lockdown=inactive for zelkor verify, so GitOps and FinServe-style installs do not break. Platform: a separate agent-dns-egress policy allows port 53 only for agent pods without dns-lockdown=on, so older and inactive agents keep DNS regardless of their own chart's NetworkPolicy setting. Co-Authored-By: Claude Opus 5.5 --- charts/zelkor-agent/templates/_helpers.tpl | 19 +++++ charts/zelkor-agent/templates/deployment.yaml | 20 ++++++ .../templates/networkpolicies.yaml | 2 + charts/zelkor-agent/values.schema.json | 29 ++++++++ charts/zelkor-agent/values.yaml | 8 +++ .../templates/security/networkpolicies.yaml | 32 +++++++++ tests/test_helm_agent_dns_lockdown.py | 69 +++++++++++++++++++ tests/test_helm_dns_lockdown.py | 5 ++ 8 files changed, 184 insertions(+) create mode 100644 tests/test_helm_agent_dns_lockdown.py diff --git a/charts/zelkor-agent/templates/_helpers.tpl b/charts/zelkor-agent/templates/_helpers.tpl index 1135f71..7787dd9 100644 --- a/charts/zelkor-agent/templates/_helpers.tpl +++ b/charts/zelkor-agent/templates/_helpers.tpl @@ -421,3 +421,22 @@ else inherit {releaseName}-ai-gateway-consumer (zelkor deploy copies it into oth {{- if and $s $s.data -}}{{- index $s.data "agents" | default "" | sha256sum | trunc 16 -}}{{- end -}} {{- end -}} {{- end }} + +{{/* +CE-21: "true" when the pod runs without DNS: lockdown is on and something fills its hostAliases +(platform.hostAliases from `zelkor deploy`, or the platform admission policy on K8s >= 1.36). +"inactive" when lockdown is on but neither can: the pod keeps DNS and is marked for +`zelkor verify` (plain Helm / GitOps installs on older clusters must not break). "false" when off. +*/}} +{{- define "zelkor-agent.dnsLockdown" -}} +{{- $on := (((.Values.security).dnsLockdown).enabled) | default false -}} +{{- $aliases := ((.Values.platform).hostAliases) | default list -}} +{{- $map := and (.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/MutatingAdmissionPolicy") (include "zelkor-agent.platformReleaseName" .) -}} +{{- if not $on -}} +false +{{- else if or $aliases $map -}} +true +{{- else -}} +inactive +{{- end -}} +{{- end }} diff --git a/charts/zelkor-agent/templates/deployment.yaml b/charts/zelkor-agent/templates/deployment.yaml index ff84c64..15a4b92 100644 --- a/charts/zelkor-agent/templates/deployment.yaml +++ b/charts/zelkor-agent/templates/deployment.yaml @@ -46,11 +46,31 @@ spec: metadata: annotations: checksum/ai-gateway-consumer: {{ include "zelkor-agent.consumerKeyChecksum" . | quote }} + {{- if eq (include "zelkor-agent.dnsLockdown" .) "inactive" }} + # CE-21: lockdown requested, but no host map to fill hostAliases; DNS stays on. + zelkor.io/dns-lockdown: inactive + {{- end }} labels: {{- include "zelkor-agent.selectorLabels" . | nindent 8 }} zelkor.io/workload-type: agent zelkor.io/datastores: {{ include "zelkor-agent.datastores" . }} + {{- with (include "zelkor-agent.platformReleaseName" .) }} + zelkor.io/platform-release: {{ . | quote }} + {{- end }} + {{- if eq (include "zelkor-agent.dnsLockdown" .) "true" }} + zelkor.io/dns-lockdown: "on" + {{- end }} spec: + {{- if eq (include "zelkor-agent.dnsLockdown" .) "true" }} + # CE-21: no DNS. Names resolve from /etc/hosts; nothing answers on 127.0.0.1:53. + dnsPolicy: None + dnsConfig: + nameservers: ["127.0.0.1"] + {{- with ((.Values.platform).hostAliases) }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/charts/zelkor-agent/templates/networkpolicies.yaml b/charts/zelkor-agent/templates/networkpolicies.yaml index 0fdda01..4d788a5 100644 --- a/charts/zelkor-agent/templates/networkpolicies.yaml +++ b/charts/zelkor-agent/templates/networkpolicies.yaml @@ -18,6 +18,7 @@ spec: policyTypes: - Egress egress: + {{- if ne (include "zelkor-agent.dnsLockdown" .) "true" }} - to: - namespaceSelector: matchLabels: @@ -27,6 +28,7 @@ spec: port: 53 - protocol: TCP port: 53 + {{- end }} - to: - namespaceSelector: matchLabels: diff --git a/charts/zelkor-agent/values.schema.json b/charts/zelkor-agent/values.schema.json index 0c11812..bfaebdf 100644 --- a/charts/zelkor-agent/values.schema.json +++ b/charts/zelkor-agent/values.schema.json @@ -211,6 +211,14 @@ "type": "array" } } + }, + "dnsLockdown": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } } } }, @@ -275,6 +283,27 @@ "pattern": "^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$" }, "description": "CE-15a-4: platform HTTP API relays this agent may call (needs releaseName)." + }, + "hostAliases": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ip", + "hostnames" + ], + "properties": { + "ip": { + "type": "string" + }, + "hostnames": { + "type": "array", + "items": { + "type": "string" + } + } + } + } } } }, diff --git a/charts/zelkor-agent/values.yaml b/charts/zelkor-agent/values.yaml index 06cab66..ad2b98c 100644 --- a/charts/zelkor-agent/values.yaml +++ b/charts/zelkor-agent/values.yaml @@ -99,8 +99,16 @@ security: # come from the envoy-gateway-system namespace, so the ingress rule below would drop it. # `zelkor deploy` fills this on kind clusters; leave empty everywhere else. hostNetworkGatewayCIDRs: [] + # CE-21: the pod gets no DNS (dnsPolicy None, nameserver 127.0.0.1 where nothing answers) and + # reaches platform names through hostAliases. Filled by the platform's admission policy on + # K8s >= 1.36, or by `zelkor deploy` (platform.hostAliases) on older clusters. Turn off only for + # an agent that must resolve names outside the platform (e.g. its own external database). + dnsLockdown: + enabled: true platform: + # CE-21: [{ip, hostnames: [...]}] from the platform host map. `zelkor deploy` fills it. + hostAliases: [] # Platform Helm release namespace. Empty → this release namespace. # CLI sets this when `zelkor deploy --namespace` differs from the platform env. namespace: "" diff --git a/charts/zelkor-platform/templates/security/networkpolicies.yaml b/charts/zelkor-platform/templates/security/networkpolicies.yaml index cfcff76..82fd7e5 100644 --- a/charts/zelkor-platform/templates/security/networkpolicies.yaml +++ b/charts/zelkor-platform/templates/security/networkpolicies.yaml @@ -69,6 +69,38 @@ spec: port: 3000 {{- end }} {{- if .Values.aegra.enabled }} +{{- if .Values.security.dnsLockdown.enabled }} +--- +# CE-21: DNS only for agent pods that are NOT locked down (older agent charts, or lockdown +# inactive because nothing could fill their hostAliases). Locked pods carry dns-lockdown=on. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-agent-dns-egress + namespace: {{ $ns }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +spec: + podSelector: + matchLabels: + zelkor.io/workload-type: agent + matchExpressions: + - key: zelkor.io/dns-lockdown + operator: NotIn + values: ["on"] + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +{{- end }} --- # Aegra runtimes (front door + workers) need Postgres + Valkey for the checkpointer. # Agent pods that state a narrower need (zelkor.io/datastores: none | postgres; sidecar runtimes) diff --git a/tests/test_helm_agent_dns_lockdown.py b/tests/test_helm_agent_dns_lockdown.py new file mode 100644 index 0000000..b642ff7 --- /dev/null +++ b/tests/test_helm_agent_dns_lockdown.py @@ -0,0 +1,69 @@ +"""CE-21 render contract: agent pods without DNS (agent chart side).""" +from __future__ import annotations + +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +CHART = ROOT / "charts" / "zelkor-agent" +MAP_API = "admissionregistration.k8s.io/v1/MutatingAdmissionPolicy" +BASE = ["--set", "graphId=g", "--set", "image.repository=ghcr.io/acme/agent", "--set-string", "image.tag=1.0.0", + "--set", "platform.releaseName=zelkor-platform", "--set", "security.networkPolicies.enabled=true"] +ALIASES = ["--set", "platform.hostAliases[0].ip=10.43.175.236", + "--set", "platform.hostAliases[0].hostnames[0]=zelkor-platform-ai-gateway", + "--set", "platform.hostAliases[0].hostnames[1]=zelkor-platform-mcp"] + + +def _docs(*extra: str) -> list[dict]: + proc = subprocess.run(["helm", "template", "a", str(CHART), *BASE, *extra], + check=False, capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return [d for d in yaml.safe_load_all(proc.stdout) if d] + + +def _pod(docs: list[dict]) -> dict: + return next(d for d in docs if d["kind"] == "Deployment")["spec"]["template"] + + +def _egress_allows_dns(docs: list[dict]) -> bool: + np = next(d for d in docs if d["kind"] == "NetworkPolicy" and d["metadata"]["name"].endswith("-egress")) + return any(p.get("port") == 53 for r in np["spec"]["egress"] for p in r.get("ports", [])) + + +def test_host_aliases_from_zelkor_deploy_lock_the_pod(): + docs = _docs(*ALIASES) + pod = _pod(docs) + assert pod["spec"]["dnsPolicy"] == "None" + assert pod["spec"]["dnsConfig"] == {"nameservers": ["127.0.0.1"]} + assert pod["spec"]["hostAliases"] == [{"ip": "10.43.175.236", + "hostnames": ["zelkor-platform-ai-gateway", "zelkor-platform-mcp"]}] + assert pod["metadata"]["labels"]["zelkor.io/dns-lockdown"] == "on" + assert pod["metadata"]["labels"]["zelkor.io/platform-release"] == "zelkor-platform" + assert not _egress_allows_dns(docs) + + +def test_admission_policy_cluster_locks_without_host_aliases(): + docs = _docs("--api-versions", MAP_API) + pod = _pod(docs) + assert pod["spec"]["dnsPolicy"] == "None" and "hostAliases" not in pod["spec"] + assert pod["metadata"]["labels"]["zelkor.io/dns-lockdown"] == "on" + assert not _egress_allows_dns(docs) + + +def test_plain_helm_on_an_older_cluster_keeps_dns_and_is_marked(): + docs = _docs() + pod = _pod(docs) + assert "dnsPolicy" not in pod["spec"] + assert "zelkor.io/dns-lockdown" not in pod["metadata"]["labels"] + assert pod["metadata"]["annotations"]["zelkor.io/dns-lockdown"] == "inactive" + assert _egress_allows_dns(docs) + + +def test_lockdown_off_keeps_dns_unmarked(): + docs = _docs("--set", "security.dnsLockdown.enabled=false", *ALIASES) + pod = _pod(docs) + assert "dnsPolicy" not in pod["spec"] and "hostAliases" not in pod["spec"] + assert "zelkor.io/dns-lockdown" not in (pod["metadata"].get("annotations") or {}) + assert _egress_allows_dns(docs) diff --git a/tests/test_helm_dns_lockdown.py b/tests/test_helm_dns_lockdown.py index 3827116..ad64121 100644 --- a/tests/test_helm_dns_lockdown.py +++ b/tests/test_helm_dns_lockdown.py @@ -54,6 +54,11 @@ def test_default_writes_the_host_map_and_agent_egress_has_no_dns(): # A plain Job, never a hook: Helm must not wait on it. assert "helm.sh/hook" not in (job["metadata"].get("annotations") or {}) assert not _allows_dns(_agent_egress(docs)) + # Agents not locked down (older charts, lockdown inactive) keep DNS through their own policy. + dns = next(d for d in _kind(docs, "NetworkPolicy") if d["metadata"]["name"] == "zelkor-platform-agent-dns-egress") + assert dns["spec"]["podSelector"]["matchExpressions"] == [ + {"key": "zelkor.io/dns-lockdown", "operator": "NotIn", "values": ["on"]}] + assert _allows_dns(dns) def test_writer_rbac_is_namespaced_and_scoped_to_its_configmap(): From 13e29f0da346ed856844a6307d8e8771ace626da Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 15:48:27 +0100 Subject: [PATCH 04/10] feat(cli): zelkor deploy fills agent hostAliases; verify --hosts checks them The platform publishes AGENT_HOSTS_CONFIGMAP in its discovery contract. deploy and deploy -f wait (ZELKOR_HOST_MAP_WAIT_SEC, 120 s) for that map, group hostnames per IP and pass them as platform.hostAliases; a platform without the key is skipped. zelkor verify --hosts reports map drift against live Services, pods whose hostAliases differ from the map, sidecar HTTP API names with no platform route (the stale api-echo case), and agents still on DNS as WARN. Co-Authored-By: Claude Opus 5.5 --- .../templates/contract-configmap.yaml | 4 + cli/src/zelkor/hosts_check.py | 121 +++++++++++++++++ cli/src/zelkor/main.py | 55 ++++++++ docs/cli.md | 6 + tests/test_zelkor_hosts.py | 128 ++++++++++++++++++ 5 files changed, 314 insertions(+) create mode 100644 cli/src/zelkor/hosts_check.py create mode 100644 tests/test_zelkor_hosts.py diff --git a/charts/zelkor-platform/templates/contract-configmap.yaml b/charts/zelkor-platform/templates/contract-configmap.yaml index d46b213..74a0167 100644 --- a/charts/zelkor-platform/templates/contract-configmap.yaml +++ b/charts/zelkor-platform/templates/contract-configmap.yaml @@ -37,6 +37,10 @@ data: {{- /* CE-15a-4: HTTP APIs relayed as {release}-api-{name}. */}} HTTP_APIS: {{ join "," $apis | quote }} {{- end }} + {{- if .Values.security.dnsLockdown.enabled }} + {{- /* CE-21: agent pods get no DNS; their hostAliases come from this host map. */}} + AGENT_HOSTS_CONFIGMAP: {{ printf "%s-agent-hosts" $full | quote }} + {{- end }} {{- if .Values.gateway.enabled }} GATEWAY_NAME: {{ $pr.name | default (printf "%s-gateway" $full) | quote }} GATEWAY_NAMESPACE: {{ $pr.namespace | default $ns | quote }} diff --git a/cli/src/zelkor/hosts_check.py b/cli/src/zelkor/hosts_check.py new file mode 100644 index 0000000..d38309c --- /dev/null +++ b/cli/src/zelkor/hosts_check.py @@ -0,0 +1,121 @@ +"""CE-21 `zelkor verify --hosts`: agent pods without DNS still reach what they need. + +Checks, each printed as PASS / FAIL / WARN: + 1. every host-map entry still points at the live ClusterIP (ExternalName followed); + 2. every running agent pod of this platform carries exactly the map's entries; + 3. every HTTP API an agent's sidecar relays to exists on the platform (no stale route names); + 4. agents left with DNS (lockdown inactive) are reported. +""" +from __future__ import annotations + +import json +import logging +import sys +from typing import Any, Callable, Optional + +logger = logging.getLogger("zelkor-cli") + +RunFn = Callable[..., Any] +CLUSTER_SUFFIX = ".svc.cluster.local" + + +def _json(runner: RunFn, argv: list[str]) -> Optional[dict]: + res = runner(argv, capture_output=True, text=True, check=False) + if res.returncode != 0: + return None + try: + return json.loads(res.stdout or "{}") + except json.JSONDecodeError: + return None + + +def _service_ip(runner: RunFn, kubectl: list[str], name: str, namespace: str, hops: int = 3) -> str: + for _ in range(hops + 1): + svc = _json(runner, kubectl + ["get", "svc", name, "-n", namespace, "-o", "json"]) + if not svc: + return "" + spec = svc.get("spec") or {} + if spec.get("type") == "ExternalName": + target = str(spec.get("externalName") or "").rstrip(".") + if not target.endswith(CLUSTER_SUFFIX): + return "" + parts = target[: -len(CLUSTER_SUFFIX)].split(".") + name, namespace = parts[0], (parts[1] if len(parts) > 1 else namespace) + continue + ip = str(spec.get("clusterIP") or "") + return "" if ip == "None" else ip + return "" + + +def _short(host: str, namespace: str) -> str: + suffix = f".{namespace}{CLUSTER_SUFFIX}" + return host[: -len(suffix)] if host.endswith(suffix) else host + + +def _pod_aliases(pod: dict) -> dict[str, str]: + out: dict[str, str] = {} + for entry in ((pod.get("spec") or {}).get("hostAliases") or []): + for host in entry.get("hostnames") or []: + out[str(host)] = str(entry.get("ip") or "") + return out + + +def _api_upstreams(pod: dict) -> list[str]: + names: list[str] = [] + for container in ((pod.get("spec") or {}).get("containers") or []): + for env in container.get("env") or []: + if env.get("name") == "ZELKOR_API_UPSTREAMS": + for item in str(env.get("value") or "").split(","): + if "=" in item: + names.append(item.split("=", 1)[0].strip()) + return names + + +def check_agent_hosts(release: str, namespace: str, *, kubectl: list[str], runner: RunFn, + contract_configmap: str = "zelkor-platform-contract") -> int: + contract = _json(runner, kubectl + ["get", "configmap", contract_configmap, "-n", namespace, "-o", "json"]) + contract_data = (contract or {}).get("data") or {} + cm_name = contract_data.get("AGENT_HOSTS_CONFIGMAP", "") + if not cm_name: + print(f"SKIP hosts {namespace}/{release} publishes no AGENT_HOSTS_CONFIGMAP (lockdown off or platform predates CE-21)") + return 0 + cm = _json(runner, kubectl + ["get", "configmap", cm_name, "-n", namespace, "-o", "json"]) + host_map: dict[str, str] = {str(k): str(v) for k, v in ((cm or {}).get("data") or {}).items()} + failed = 0 + if not host_map: + print(f"FAIL host map {cm_name} missing or empty; check job/{release}-agent-hosts-") + return 1 + + for host in sorted(h for h in host_map if not h.endswith(CLUSTER_SUFFIX)): + live = _service_ip(runner, kubectl, host, namespace) + ok = live == host_map[host] + failed += 0 if ok else 1 + print(f"{'PASS' if ok else 'FAIL'} host map {host} -> {host_map[host]}" + + ("" if ok else f" (live: {live or 'missing'}; upgrade the platform to rewrite the map)")) + + apis = {n.strip() for n in contract_data.get("HTTP_APIS", "").split(",") if n.strip()} + pods = _json(runner, kubectl + ["get", "pods", "-A", "-l", "zelkor.io/workload-type=agent", "-o", "json"]) or {} + for pod in pods.get("items") or []: + meta = pod.get("metadata") or {} + where = f"{meta.get('namespace')}/{meta.get('name')}" + labels = meta.get("labels") or {} + if labels.get("zelkor.io/platform-release", release) != release: + continue + if (meta.get("annotations") or {}).get("zelkor.io/dns-lockdown") == "inactive": + print(f"WARN dns {where} keeps DNS (lockdown inactive); redeploy with `zelkor deploy`") + elif labels.get("zelkor.io/dns-lockdown") == "on": + have = {_short(h, namespace): ip for h, ip in _pod_aliases(pod).items()} + want = {_short(h, namespace): ip for h, ip in host_map.items()} + drift = sorted(h for h in set(have) | set(want) if have.get(h) != want.get(h)) + failed += 1 if drift else 0 + print(f"{'FAIL' if drift else 'PASS'} pod hosts {where}" + + (f" differs on {', '.join(drift)}; delete the pod (K8s >= 1.36) or run `zelkor deploy`" if drift else "")) + for name in _api_upstreams(pod): + ok = name in apis + failed += 0 if ok else 1 + print(f"{'PASS' if ok else 'FAIL'} http api {where} -> {release}-api-{name}" + + ("" if ok else " has no route on the platform; redeploy the agent or add the API")) + logger.info("verify hosts release=%s failed=%s", release, failed) + print(f"{'FAILED' if failed else 'OK'}: {failed} problem(s) in the agent host checks for {namespace}/{release}", + file=sys.stderr if failed else sys.stdout) + return 1 if failed else 0 diff --git a/cli/src/zelkor/main.py b/cli/src/zelkor/main.py index 50f0afe..730e8a9 100644 --- a/cli/src/zelkor/main.py +++ b/cli/src/zelkor/main.py @@ -36,6 +36,7 @@ from zelkor.extra_backends import format_missing_extras_error, missing_extra_registrations from zelkor.token_cmd import cmd_token_jwks, cmd_token_mint from zelkor.verify_cmd import cmd_verify +from zelkor.hosts_check import check_agent_hosts PAID = frozenset({"login", "license", "whoami", "team", "budget", "audit"}) UPGRADE = "This command requires Zelkor Pro or Enterprise. Community Edition does not apply it." @@ -82,6 +83,8 @@ class PlatformInfo: network_policies_enabled: bool = False gateway_host_cidrs: list[str] = field(default_factory=list) http_apis: list[str] = field(default_factory=list) # CE-15a-4 relays {release}-api-{name} + agent_hosts_configmap: str = "" # CE-21 contract AGENT_HOSTS_CONFIGMAP (empty: platform predates it) + host_aliases: list[dict[str, Any]] = field(default_factory=list) # CE-21 [{ip, hostnames}] def find_chart(start: Path, chart_name: str, explicit: str = "", env_key: str = "") -> Path: @@ -287,6 +290,7 @@ def discover_platform(env: Env, runner: Optional[RunFn] = None) -> PlatformInfo: if contract.get("PROFILE"): info.profile = contract["PROFILE"].strip().lower() info.http_apis = [n.strip() for n in contract.get("HTTP_APIS", "").split(",") if n.strip()] + info.agent_hosts_configmap = contract.get("AGENT_HOSTS_CONFIGMAP", "") info.aegra_present = bool(contract.get("AEGRA_FRONTDOOR_SERVICE")) info.mcp_service = contract.get("MCP_SERVICE", "") info.agents_host = contract.get("AGENTS_HOST", "") @@ -539,6 +543,47 @@ def _apply_stateless_wrap(overlay: dict[str, Any], info: PlatformInfo) -> None: platform["otlp"] = True +HOST_MAP_WAIT_SEC = float(os.getenv("ZELKOR_HOST_MAP_WAIT_SEC", "120")) + + +def host_aliases_from_map(data: dict[str, str]) -> list[dict[str, Any]]: + """CE-21 host map (hostname -> IP) as pod hostAliases, one entry per IP, stable order.""" + by_ip: dict[str, list[str]] = {} + for host, ip in data.items(): + if host and ip: + by_ip.setdefault(ip.strip(), []).append(host.strip()) + return [{"ip": ip, "hostnames": sorted(names)} for ip, names in sorted(by_ip.items())] + + +def load_host_aliases(env: Env, info: PlatformInfo, *, runner: Optional[RunFn] = None, + wait_sec: Optional[float] = None, sleep=time.sleep) -> None: + """CE-21: fill info.host_aliases from the platform host map, waiting for its writer Job. + + A platform that publishes no AGENT_HOSTS_CONFIGMAP predates CE-21 (or turned lockdown off): + the agent keeps DNS and nothing is waited for. + """ + if not info.agent_hosts_configmap: + return + deadline = time.monotonic() + (HOST_MAP_WAIT_SEC if wait_sec is None else wait_sec) + while True: + data = _configmap_data(env, info.agent_hosts_configmap, runner=runner) + if data: + info.host_aliases = host_aliases_from_map(data) + return + if time.monotonic() >= deadline: + raise RuntimeError( + f"host map {info.agent_hosts_configmap} not written yet; check the platform Job " + f"{info.release}-agent-hosts- (kubectl -n {env.namespace} logs job/...)" + ) + logger.info("waiting for host map %s", info.agent_hosts_configmap) + sleep(5) + + +def _apply_host_aliases(overlay: dict[str, Any], info: PlatformInfo) -> None: + if info.host_aliases: + overlay.setdefault("platform", {})["hostAliases"] = list(info.host_aliases) + + def _apply_http_apis(overlay: dict[str, Any], info: PlatformInfo) -> None: """CE-15a-4: the sidecar may relay to every platform HTTP API (credential stays in the gateway).""" if info.http_apis and overlay.get("runtime", "aegra") != "aegra": @@ -1410,6 +1455,8 @@ def deploy_agent( if egress_only: _apply_stateless_wrap(overlay, info) _apply_http_apis(overlay, info) + load_host_aliases(env, info, runner=runner) + _apply_host_aliases(overlay, info) if run_timeout or max_tokens or approval_threshold: overlay["workload"] = { "intent": { @@ -1537,6 +1584,8 @@ def deploy_from_values( if stateless: _apply_stateless_wrap(overlay, info) _apply_http_apis(overlay, info) + load_host_aliases(env, info, runner=runner) + _apply_host_aliases(overlay, info) release = helm_release_name(graph_id) # A catalog entry is a worker unless its file asks for `/`. as_default, takes_over_front_door = resolve_default_route( @@ -1936,6 +1985,9 @@ def main(argv: Optional[list[str]] = None, runner: Optional[RunFn] = None) -> in p_verify = sub.add_parser("verify", help="cosign-verify the Zelkor images a Helm release runs") p_verify.add_argument("release", help="Helm release name (platform or agent)") p_verify.add_argument("--namespace", default="", help="release namespace (default: the env namespace)") + p_verify.add_argument("--hosts", action="store_true", + help="check agent pods without DNS (host map, pod hostAliases, HTTP API routes) " + "instead of image signatures; release = the platform release") p_token = sub.add_parser("token", help="mint RS256 tokens from cluster signing key") token_sub = p_token.add_subparsers(dest="token_cmd", required=True) @@ -2061,6 +2113,9 @@ def main(argv: Optional[list[str]] = None, runner: Optional[RunFn] = None) -> in return cmd_doctor(env, runner=runner) if args.cmd == "verify": target = env_for_namespace(env, args.namespace) if args.namespace else env + if args.hosts: + return check_agent_hosts(args.release, target.namespace, kubectl=kube_argv(target), + runner=runner or subprocess.run, contract_configmap=CONTRACT_CONFIGMAP) return cmd_verify( args.release, target.namespace, diff --git a/docs/cli.md b/docs/cli.md index cbdf38e..562cb4a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -109,6 +109,12 @@ zelkor verify desk --namespace agents `verify` lists the Zelkor images running in the release's pods and runs `cosign verify` and `cosign verify-attestation --type slsaprovenance1` on each digest, bound to the Zelkor release workflow identity. It exits non-zero if any check fails or no Zelkor image runs. Needs [cosign](https://docs.sigstore.dev/cosign/system_config/installation/) on your PATH. See [Verify a Release](security/verify-release.md). +```bash +zelkor verify zelkor-platform --hosts +``` + +`--hosts` checks agents that run without DNS instead of image signatures, so it needs no cosign. It reports `FAIL` when a host-map entry no longer matches its Service, when a running agent pod carries other entries than the map, or when an agent's sidecar relays to an HTTP API the platform no longer has. Agents that still have DNS show as `WARN`. To fix pod entries, delete the pod (Kubernetes 1.36 and later) or run `zelkor deploy` again. + ## Global flags | Flag | Purpose | diff --git a/tests/test_zelkor_hosts.py b/tests/test_zelkor_hosts.py new file mode 100644 index 0000000..516f111 --- /dev/null +++ b/tests/test_zelkor_hosts.py @@ -0,0 +1,128 @@ +"""CE-21 CLI: zelkor deploy fills hostAliases from the host map; zelkor verify --hosts finds drift.""" +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "cli" / "src")) +from zelkor.envfile import Env # noqa: E402 +from zelkor.hosts_check import check_agent_hosts # noqa: E402 +from zelkor.main import (PlatformInfo, _apply_host_aliases, host_aliases_from_map, # noqa: E402 + load_host_aliases) + +ENV = Env(name="l", kube_context="kind-zelkor", namespace="zelkor") +GW = "10.43.175.236" +MAP = {"zelkor-platform-ai-gateway": GW, "zelkor-platform-mcp": GW, + "zelkor-platform-ai-gateway.zelkor.svc.cluster.local": GW, "zelkor-platform-postgresql": "10.43.217.227"} + + +def _ok(obj): + return SimpleNamespace(returncode=0, stdout=json.dumps(obj), stderr="") + + +MISSING = SimpleNamespace(returncode=1, stdout="", stderr="NotFound") + + +def test_map_groups_hostnames_per_ip_in_stable_order(): + assert host_aliases_from_map(MAP) == [ + {"ip": GW, "hostnames": ["zelkor-platform-ai-gateway", "zelkor-platform-ai-gateway.zelkor.svc.cluster.local", + "zelkor-platform-mcp"]}, + {"ip": "10.43.217.227", "hostnames": ["zelkor-platform-postgresql"]}, + ] + + +def test_platform_without_contract_key_is_skipped(): + info = PlatformInfo(release="zelkor-platform") + load_host_aliases(ENV, info, runner=lambda *a, **k: pytest.fail("must not read"), wait_sec=0) + overlay: dict = {} + _apply_host_aliases(overlay, info) + assert overlay == {} + + +def test_deploy_waits_for_the_map_then_fills_platform_host_aliases(): + info = PlatformInfo(release="zelkor-platform", agent_hosts_configmap="zelkor-platform-agent-hosts") + calls = {"n": 0} + + def runner(argv, **_k): + calls["n"] += 1 + return MISSING if calls["n"] == 1 else _ok({"data": MAP}) + + load_host_aliases(ENV, info, runner=runner, wait_sec=60, sleep=lambda _s: None) + overlay: dict = {"platform": {"releaseName": "zelkor-platform"}} + _apply_host_aliases(overlay, info) + assert calls["n"] == 2 + assert overlay["platform"]["hostAliases"][0]["ip"] == GW + + +def test_deploy_fails_naming_the_writer_job_when_the_map_never_appears(): + info = PlatformInfo(release="zelkor-platform", agent_hosts_configmap="zelkor-platform-agent-hosts") + with pytest.raises(RuntimeError, match="zelkor-platform-agent-hosts-"): + load_host_aliases(ENV, info, runner=lambda *a, **k: MISSING, wait_sec=0, sleep=lambda _s: None) + + +def _cluster(*, live_gw=GW, pod_aliases=None, upstreams="", annotations=None, lockdown="on"): + contract = {"data": {"AGENT_HOSTS_CONFIGMAP": "zelkor-platform-agent-hosts", "HTTP_APIS": "github"}} + services = { + "zelkor-platform-ai-gateway": {"spec": {"type": "ExternalName", + "externalName": "dp.envoy-gateway-system.svc.cluster.local"}}, + "zelkor-platform-mcp": {"spec": {"type": "ExternalName", + "externalName": "dp.envoy-gateway-system.svc.cluster.local"}}, + "dp": {"spec": {"type": "ClusterIP", "clusterIP": live_gw}}, + "zelkor-platform-postgresql": {"spec": {"type": "ClusterIP", "clusterIP": "10.43.217.227"}}, + } + labels = {"zelkor.io/workload-type": "agent", "zelkor.io/platform-release": "zelkor-platform"} + if lockdown: + labels["zelkor.io/dns-lockdown"] = lockdown + env = [{"name": "ZELKOR_API_UPSTREAMS", "value": upstreams}] if upstreams else [] + pod = {"metadata": {"namespace": "zelkor", "name": "rt-pydantic-x", "labels": labels, + "annotations": annotations or {}}, + "spec": {"hostAliases": pod_aliases if pod_aliases is not None else host_aliases_from_map(MAP), + "containers": [{"name": "sidecar", "env": env}]}} + + def runner(argv, **_k): + if "configmap" in argv: + name = argv[argv.index("configmap") + 1] + return _ok(contract) if name == "zelkor-platform-contract" else _ok({"data": MAP}) + if "svc" in argv: + svc = services.get(argv[argv.index("svc") + 1]) + return _ok(svc) if svc else MISSING + if "pods" in argv: + return _ok({"items": [pod]}) + return MISSING + + return runner + + +def _check(runner): + return check_agent_hosts("zelkor-platform", "zelkor", kubectl=["kubectl"], runner=runner) + + +def test_healthy_lockdown_passes(capsys): + assert _check(_cluster(upstreams="github=http://zelkor-platform-api-github")) == 0 + assert "FAIL" not in capsys.readouterr().out + + +def test_recreated_gateway_service_is_map_drift(capsys): + assert _check(_cluster(live_gw="10.43.9.9")) == 1 + assert "live: 10.43.9.9" in capsys.readouterr().out + + +def test_pod_with_old_entries_is_drift(capsys): + stale = [{"ip": "10.43.1.1", "hostnames": ["zelkor-platform-ai-gateway"]}] + assert _check(_cluster(pod_aliases=stale)) == 1 + assert "FAIL pod hosts" in capsys.readouterr().out + + +def test_stale_http_api_name_is_reported(capsys): + # demohetzner 2026-11-18: rt-pydantic still relayed to api-echo after the platform dropped it. + assert _check(_cluster(upstreams="echo=http://zelkor-platform-api-echo,github=http://x")) == 1 + out = capsys.readouterr().out + assert "zelkor-platform-api-echo has no route" in out and "PASS http api" in out + + +def test_inactive_lockdown_is_a_warning_not_a_failure(capsys): + assert _check(_cluster(lockdown="", annotations={"zelkor.io/dns-lockdown": "inactive"})) == 0 + assert "WARN dns" in capsys.readouterr().out From c7bd637e0db670e88c358fcb7b8f5ebb03c96dfd Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 15:50:24 +0100 Subject: [PATCH 05/10] feat(agents): sidecar readiness fails while a gateway upstream is unreachable CE-21 agent pods resolve names from hostAliases; a recreated Service leaves a stale entry. The sidecar now TCP-checks its LLM and MCP upstreams (native resolution, cached 10 s) as part of /ready. On a transition to unreachable it logs one ERROR upstream_unreachable with the upstream, host, IP and a hint to run zelkor deploy (or delete the pod on K8s >= 1.36); recovery logs one INFO upstream_reachable. Co-Authored-By: Claude Opus 5.5 --- images/sidecar/zelkor_sidecar/egress.py | 60 +++++++++++++++++++ images/sidecar/zelkor_sidecar/main.py | 13 +++- tests/test_sidecar_upstream_probe.py | 80 +++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 tests/test_sidecar_upstream_probe.py diff --git a/images/sidecar/zelkor_sidecar/egress.py b/images/sidecar/zelkor_sidecar/egress.py index 0c21cd1..56f5712 100644 --- a/images/sidecar/zelkor_sidecar/egress.py +++ b/images/sidecar/zelkor_sidecar/egress.py @@ -15,11 +15,13 @@ import json import logging import os +import socket import time import uuid from collections import OrderedDict from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import urlparse from .runs import RunContext, RunSlot from .tracing import Tracing, set_span_error @@ -67,6 +69,64 @@ def from_env(cls) -> "EgressConfig": ) +class UpstreamProbe: + """CE-21: readiness needs the gateway upstreams reachable. + + The pod runs without DNS; names resolve from /etc/hosts (hostAliases). A recreated Service + leaves a stale entry, so the sidecar says which upstream it cannot reach instead of failing + every run silently. One ERROR per transition to unreachable, one INFO on recovery; results + are cached so a probe storm does not open a connection per request. + """ + + def __init__(self, cfg: "EgressConfig", *, cache_s: float = 10.0, timeout_s: float = 2.0, + connect=None, resolve=None, clock=time.monotonic) -> None: + self.targets = {name: url for name, url in (("llm", cfg.llm_upstream), ("mcp", cfg.mcp_upstream)) if url} + self.cache_s, self.timeout_s, self.clock = cache_s, timeout_s, clock + self._connect = connect or self._tcp_connect + self._resolve = resolve or self._getaddrinfo + self._checked_at = -1e9 + self._ok = True + self._down: set = set() + + @staticmethod + async def _getaddrinfo(host: str, port: int) -> str: + infos = await asyncio.get_running_loop().getaddrinfo(host, port, type=socket.SOCK_STREAM) + return infos[0][4][0] if infos else "" + + async def _tcp_connect(self, ip: str, port: int) -> None: + _reader, writer = await asyncio.wait_for(asyncio.open_connection(ip, port), self.timeout_s) + writer.close() + + async def _reachable(self, name: str, url: str) -> bool: + parsed = urlparse(url) + host = parsed.hostname or "" + port = parsed.port or (443 if parsed.scheme == "https" else 80) + ip = "" + try: + ip = await self._resolve(host, port) + await self._connect(ip, port) + except Exception as exc: # noqa: BLE001 - any failure means not reachable + if name not in self._down: + self._down.add(name) + logger.error("upstream unreachable", extra={ + "event": "upstream_unreachable", "upstream": name, "host": host, "ip": ip or "unresolved", + "error": type(exc).__name__, + "hint": "host entries may be stale: run zelkor deploy (or delete the pod on K8s >= 1.36)"}) + return False + if name in self._down: + self._down.discard(name) + logger.info("upstream reachable again", extra={"event": "upstream_reachable", "upstream": name}) + return True + + async def check(self) -> bool: + now = self.clock() + if now - self._checked_at < self.cache_s: + return self._ok + results = [await self._reachable(n, u) for n, u in self.targets.items()] + self._ok, self._checked_at = all(results), now + return self._ok + + def parse_http_apis(raw: str) -> Dict[str, str]: """``name=http://{release}-api-{name},...`` → {name: base URL}.""" out: Dict[str, str] = {} diff --git a/images/sidecar/zelkor_sidecar/main.py b/images/sidecar/zelkor_sidecar/main.py index 2eebc39..e62afe8 100644 --- a/images/sidecar/zelkor_sidecar/main.py +++ b/images/sidecar/zelkor_sidecar/main.py @@ -40,7 +40,7 @@ def build_store(store_mode: str | None = None): async def serve() -> None: import uvicorn - from .egress import Egress, EgressConfig + from .egress import Egress, EgressConfig, UpstreamProbe from .inbound import Inbound, graph_id_from_env from .runs import RunExecutor, RunSlot from .tracing import Tracing @@ -55,13 +55,20 @@ async def serve() -> None: tracing = Tracing.from_env() slot = RunSlot() executor = RunExecutor(store, bridge, slot, tracing, runtime=runtime) - inbound = Inbound(store, executor, graph_id=graph_id_from_env(), ready=bridge.ready) + egress_cfg = EgressConfig.from_env() + upstreams = UpstreamProbe(egress_cfg) + + async def ready() -> bool: + # CE-21: not ready while a gateway upstream is unreachable (stale host entries). + return await bridge.ready() and await upstreams.check() + + inbound = Inbound(store, executor, graph_id=graph_id_from_env(), ready=ready) apps = [inbound.app()] if os.getenv("ZELKOR_A2A_ENABLED", "false").strip().lower() in ("1", "true", "yes"): from .a2a_server import mount_a2a mount_a2a(apps[0], inbound) - egress = Egress(slot, tracing, EgressConfig.from_env()) + egress = Egress(slot, tracing, egress_cfg) log_config = None # zelkor_logging owns the root logger servers = [ uvicorn.Server(uvicorn.Config(apps[0], host="0.0.0.0", port=int(os.getenv("ZELKOR_SIDECAR_PORT", "8000")), diff --git a/tests/test_sidecar_upstream_probe.py b/tests/test_sidecar_upstream_probe.py new file mode 100644 index 0000000..f3405e9 --- /dev/null +++ b/tests/test_sidecar_upstream_probe.py @@ -0,0 +1,80 @@ +"""CE-21: the sidecar is not ready while a gateway upstream is unreachable, and says which one.""" +import asyncio +import logging +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "images" / "sidecar")) +from zelkor_sidecar.egress import EgressConfig, UpstreamProbe # noqa: E402 + +CFG = EgressConfig(llm_upstream="http://zelkor-platform-ai-gateway:80/v1", mcp_upstream="http://zelkor-platform-mcp/mcp") + + +class Net: + def __init__(self): + self.down = set() + self.connects = 0 + + async def resolve(self, host, port): + return {"zelkor-platform-ai-gateway": "10.43.175.236", "zelkor-platform-mcp": "10.43.175.236"}[host] + + async def connect(self, ip, port): + self.connects += 1 + if port in self.down: + raise ConnectionRefusedError() + + +def _probe(net, now): + return UpstreamProbe(CFG, resolve=net.resolve, connect=net.connect, clock=lambda: now[0]) + + +def test_ready_when_both_upstreams_connect(): + net, now = Net(), [0.0] + assert asyncio.run(_probe(net, now).check()) is True + assert net.connects == 2 + + +def test_unreachable_upstream_fails_readiness_and_logs_once(caplog): + net, now = Net(), [0.0] + probe = _probe(net, now) + net.down.add(80) + with caplog.at_level(logging.INFO, logger="zelkor-sidecar"): + assert asyncio.run(probe.check()) is False + now[0] = 30.0 + assert asyncio.run(probe.check()) is False + errors = [r for r in caplog.records if getattr(r, "event", "") == "upstream_unreachable"] + # Both upstreams use port 80 here; each reports once, not once per probe. + assert sorted(r.upstream for r in errors) == ["llm", "mcp"] + assert all(r.levelno == logging.ERROR and r.ip == "10.43.175.236" and "zelkor deploy" in r.hint for r in errors) + + +def test_recovery_is_logged_and_ready_again(caplog): + net, now = Net(), [0.0] + probe = _probe(net, now) + net.down.add(80) + asyncio.run(probe.check()) + net.down.clear() + now[0] = 30.0 + with caplog.at_level(logging.INFO, logger="zelkor-sidecar"): + assert asyncio.run(probe.check()) is True + assert {r.upstream for r in caplog.records if getattr(r, "event", "") == "upstream_reachable"} == {"llm", "mcp"} + + +def test_results_are_cached_between_probes(): + net, now = Net(), [0.0] + probe = _probe(net, now) + asyncio.run(probe.check()) + now[0] = 5.0 + asyncio.run(probe.check()) + assert net.connects == 2 + + +def test_unresolvable_name_reports_unresolved(caplog): + async def no_name(host, port): + raise OSError("no such host") + + probe = UpstreamProbe(CFG, resolve=no_name, connect=Net().connect, clock=lambda: 0.0) + with caplog.at_level(logging.ERROR, logger="zelkor-sidecar"): + assert asyncio.run(probe.check()) is False + assert {r.ip for r in caplog.records} == {"unresolved"} From 36c6a9ac422b02aef0b331b685b56f85112c08e5 Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 15:53:32 +0100 Subject: [PATCH 06/10] test(tests): add S-90 live DNS egress test and map it to egress-dns For every running agent pod with zelkor.io/dns-lockdown=on, the probe runs in its sidecar: a never-seen outside name and github.com must not resolve, kube-dns must not answer over UDP or TCP 53, and the gateway name must still resolve from /etc/hosts (positive control). The fail-closed matrix maps S-90 to egress-dns, so that row can turn PASS. Co-Authored-By: Claude Opus 5.5 --- scripts/release/fail_closed_categories.yaml | 4 +- tests/test_release_fail_closed_matrix.py | 7 ++ tests/test_s90_dns_egress_live.py | 115 ++++++++++++++++++++ 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 tests/test_s90_dns_egress_live.py diff --git a/scripts/release/fail_closed_categories.yaml b/scripts/release/fail_closed_categories.yaml index 709f53c..dcf92af 100644 --- a/scripts/release/fail_closed_categories.yaml +++ b/scripts/release/fail_closed_categories.yaml @@ -17,8 +17,8 @@ categories: files: [test_sidecar_egress, test_helm_agent_egress_scope] egress-dns: title: Egress via DNS (resolution and exfiltration over DNS) - s_ids: [] - files: [] + s_ids: [S-90] + files: [test_s90_dns_egress_live] admin-plane: title: Admin plane unreachable from the data plane s_ids: [S-08, S-10, S-11, S-87] diff --git a/tests/test_release_fail_closed_matrix.py b/tests/test_release_fail_closed_matrix.py index d5824cb..7fb49df 100644 --- a/tests/test_release_fail_closed_matrix.py +++ b/tests/test_release_fail_closed_matrix.py @@ -67,6 +67,13 @@ def test_categories_pass_fail_and_gap(tmp_path): assert [o.s_ids for o in report.other] == [["S-04"]] +def test_s90_turns_egress_dns_pass(tmp_path): + junit = _junit(tmp_path, [("tests.test_s90_dns_egress_live", "test_s90_locked_agents_cannot_resolve_or_reach_dns", + "passed")]) + report = fcm.build(fcm.load_mapping(MAPPING), fcm.parse_junit(junit)) + assert {c.key: c.status for c in report.categories}["egress-dns"] == "PASS" + + def test_error_counts_as_failure(tmp_path): junit = _junit(tmp_path, [("tests.test_s09_agent_egress", "a", "passed"), ("tests.test_s09_agent_egress", "b", "error")]) diff --git a/tests/test_s90_dns_egress_live.py b/tests/test_s90_dns_egress_live.py new file mode 100644 index 0000000..f613b64 --- /dev/null +++ b/tests/test_s90_dns_egress_live.py @@ -0,0 +1,115 @@ +"""S-90 (P8, P18; fail-closed `egress-dns`): an agent pod cannot send data out through DNS. + +Before CE-21 an agent could resolve any name through kube-dns, so data written into a query +label reached the attacker's authoritative nameserver although every direct connection was +blocked. Proof 2026-11-18 on demohetzner: a never-seen `.example.com` came back with +example.com's SOA from ns.cloudflare.com. + +Live (L3): for every locked agent pod (zelkor.io/dns-lockdown=on) the probe runs in its sidecar: +A and TXT lookups of a never-seen outside name and of github.com must fail, kube-dns must be +unreachable on UDP and TCP 53, and no answer may carry an outside SOA. Positive control: the +gateway is still reachable by name from /etc/hosts. +""" +from __future__ import annotations + +import json +import os +import subprocess + +import pytest + +from tests.helpers.kube import kubectl_cmd + +PROBE = r""" +import json, os, random, socket, struct + +def dns(name, qtype, server, tcp=False): + tid = random.randint(0, 0xFFFF) + q = struct.pack(">HHHHHH", tid, 0x0100, 1, 0, 0, 0) + q += b"".join(bytes([len(p)]) + p.encode() for p in name.split(".")) + b"\0" + struct.pack(">HH", qtype, 1) + try: + if tcp: + s = socket.create_connection((server, 53), 3) + s.sendall(struct.pack(">H", len(q)) + q) + r = s.recv(4096)[2:] + else: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(3) + s.sendto(q, (server, 53)) + r, _ = s.recvfrom(4096) + except Exception as e: + return {"answered": False, "error": type(e).__name__} + an, ns = struct.unpack(">HH", r[6:10]) + return {"answered": True, "rcode": r[3] & 0xF, "answers": an, "authority": ns} + +def resolves(name): + try: + socket.getaddrinfo(name, 443) + return True + except OSError: + return False + +kube_dns = os.environ.get("S90_KUBE_DNS", "10.96.0.10") +tag = "zelkor-s90-" + os.urandom(4).hex() + ".example.com" +out = { + "random_resolves": resolves(tag), + "github_resolves": resolves("github.com"), + "udp_a": dns(tag, 1, kube_dns), + "udp_txt": dns("example.com", 16, kube_dns), + "tcp_a": dns(tag, 1, kube_dns, tcp=True), + "gateway": resolves(os.environ.get("S90_GATEWAY_HOST", "")), +} +print("S90 " + json.dumps(out)) +""" + + +def _kubectl_json(*args: str): + res = subprocess.run(kubectl_cmd(*args, "-o", "json"), capture_output=True, text=True, timeout=60) + if res.returncode != 0: + pytest.skip(f"kubectl failed: {res.stderr.strip()[:200]}") + return json.loads(res.stdout or "{}") + + +def _kube_dns_ip() -> str: + svc = _kubectl_json("get", "svc", "kube-dns", "-n", "kube-system") + return str((svc.get("spec") or {}).get("clusterIP") or "") + + +def _locked_pods() -> list[dict]: + pods = _kubectl_json("get", "pods", "-A", "-l", "zelkor.io/dns-lockdown=on") + return [p for p in pods.get("items") or [] if (p.get("status") or {}).get("phase") == "Running"] + + +def _gateway_host(pod: dict) -> str: + for c in pod["spec"]["containers"]: + for env in c.get("env") or []: + if env.get("name") == "ZELKOR_LLM_UPSTREAM": + return str(env.get("value") or "").split("//", 1)[-1].split("/", 1)[0].split(":", 1)[0] + return "" + + +def test_s90_locked_agents_cannot_resolve_or_reach_dns(): + if not os.environ.get("KUBECONTEXT", "").strip(): + pytest.skip("KUBECONTEXT not set") + pods = _locked_pods() + if not pods: + pytest.skip("no running agent pod with zelkor.io/dns-lockdown=on (deploy one with zelkor deploy)") + kube_dns = _kube_dns_ip() + for pod in pods: + meta = pod["metadata"] + container = "sidecar" if any(c["name"] == "sidecar" for c in pod["spec"]["containers"]) else \ + pod["spec"]["containers"][0]["name"] + where = f"{meta['namespace']}/{meta['name']}" + res = subprocess.run( + kubectl_cmd("exec", "-i", "-n", meta["namespace"], meta["name"], "-c", container, "--", + "env", f"S90_KUBE_DNS={kube_dns}", f"S90_GATEWAY_HOST={_gateway_host(pod)}", + "python", "-"), + input=PROBE, capture_output=True, text=True, timeout=120) + line = next((ln for ln in res.stdout.splitlines() if ln.startswith("S90 ")), "") + assert line, f"{where}: probe did not run: {res.stderr[-300:]}" + out = json.loads(line[4:]) + assert pod["spec"].get("dnsPolicy") == "None", f"{where}: dnsPolicy {pod['spec'].get('dnsPolicy')}" + assert not out["random_resolves"] and not out["github_resolves"], f"{where}: outside names resolve: {out}" + for key in ("udp_a", "udp_txt", "tcp_a"): + assert not out[key]["answered"], f"{where}: kube-dns answered {key}: {out[key]}" + assert out["gateway"], f"{where}: positive control failed, the gateway name does not resolve from /etc/hosts" From 1835b29b89d2bce87c71f758e1956778213401c7 Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 16:14:48 +0100 Subject: [PATCH 07/10] fix(helm): keep port 53 out of aegra-egress for locked agent pods S-90 on tes2 (K8s 1.36.4): kube-dns still answered finserve-coder. Agent pods carry app.kubernetes.io/component=aegra, so a locked agent with Valkey access also matches aegra-egress, which allowed port 53. With lockdown on, that rule moves to aegra-dns-egress, which selects only Aegra-labelled pods without zelkor.io/dns-lockdown=on (the platform Aegra and unlocked agents). A render test now asserts every policy that allows port 53 excludes locked pods. Co-Authored-By: Claude Opus 5.5 --- .../templates/security/networkpolicies.yaml | 32 +++++++++++++++++++ tests/test_helm_dns_lockdown.py | 14 ++++++++ 2 files changed, 46 insertions(+) diff --git a/charts/zelkor-platform/templates/security/networkpolicies.yaml b/charts/zelkor-platform/templates/security/networkpolicies.yaml index 82fd7e5..1bbf2da 100644 --- a/charts/zelkor-platform/templates/security/networkpolicies.yaml +++ b/charts/zelkor-platform/templates/security/networkpolicies.yaml @@ -71,6 +71,36 @@ spec: {{- if .Values.aegra.enabled }} {{- if .Values.security.dnsLockdown.enabled }} --- +# CE-21: the platform Aegra and Aegra-labelled pods that are not locked down keep DNS. A locked +# agent with Valkey access matches aegra-egress too, so port 53 cannot live in that policy. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $fullname }}-aegra-dns-egress + namespace: {{ $ns }} + labels: + {{- include "zelkor-platform.labels" (dict "root" . "intent" "security.dnsLockdown") | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/component: aegra + matchExpressions: + - key: zelkor.io/dns-lockdown + operator: NotIn + values: ["on"] + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- # CE-21: DNS only for agent pods that are NOT locked down (older agent charts, or lockdown # inactive because nothing could fill their hostAliases). Locked pods carry dns-lockdown=on. apiVersion: networking.k8s.io/v1 @@ -123,6 +153,7 @@ spec: policyTypes: - Egress egress: + {{- if not .Values.security.dnsLockdown.enabled }} - to: - namespaceSelector: matchLabels: @@ -132,6 +163,7 @@ spec: port: 53 - protocol: TCP port: 53 + {{- end }} - to: - podSelector: matchLabels: diff --git a/tests/test_helm_dns_lockdown.py b/tests/test_helm_dns_lockdown.py index ad64121..473dbac 100644 --- a/tests/test_helm_dns_lockdown.py +++ b/tests/test_helm_dns_lockdown.py @@ -61,6 +61,20 @@ def test_default_writes_the_host_map_and_agent_egress_has_no_dns(): assert _allows_dns(dns) +def test_no_policy_gives_locked_pods_dns(): + """Every policy allowing port 53 excludes zelkor.io/dns-lockdown=on (S-90 caught aegra-egress).""" + docs = _docs() + for np in _kind(docs, "NetworkPolicy"): + if not _allows_dns(np): + continue + sel = np["spec"]["podSelector"] + labels = sel.get("matchLabels") or {} + if labels.get("app.kubernetes.io/component") not in ("aegra", None) and "zelkor.io/workload-type" not in labels: + continue # selects platform components only (MCP, Langfuse), never agent pods + assert {"key": "zelkor.io/dns-lockdown", "operator": "NotIn", "values": ["on"]} in \ + (sel.get("matchExpressions") or []), np["metadata"]["name"] + + def test_writer_rbac_is_namespaced_and_scoped_to_its_configmap(): docs = _docs() assert not _kind(docs, "ClusterRole") or all( From 28528455764cab6a7a7c1193e0cfed8bece7dd26 Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 16:15:27 +0100 Subject: [PATCH 08/10] test(tests): S-90 positive control uses the Aegra agent's gateway URL too Aegra-runtime agents have no sidecar and no ZELKOR_LLM_UPSTREAM; take the host from OPENAI_BASE_URL or MCP_URL, and fail clearly if none. Co-Authored-By: Claude Opus 5.5 --- tests/test_s90_dns_egress_live.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_s90_dns_egress_live.py b/tests/test_s90_dns_egress_live.py index f613b64..0d00f64 100644 --- a/tests/test_s90_dns_egress_live.py +++ b/tests/test_s90_dns_egress_live.py @@ -81,10 +81,12 @@ def _locked_pods() -> list[dict]: def _gateway_host(pod: dict) -> str: - for c in pod["spec"]["containers"]: - for env in c.get("env") or []: - if env.get("name") == "ZELKOR_LLM_UPSTREAM": - return str(env.get("value") or "").split("//", 1)[-1].split("/", 1)[0].split(":", 1)[0] + """Gateway host the pod calls: sidecar runtimes set ZELKOR_LLM_UPSTREAM, Aegra agents OPENAI_BASE_URL.""" + env = {e.get("name"): str(e.get("value") or "") for c in pod["spec"]["containers"] for e in c.get("env") or []} + for key in ("ZELKOR_LLM_UPSTREAM", "OPENAI_BASE_URL", "MCP_URL"): + url = env.get(key, "") + if "//" in url: + return url.split("//", 1)[1].split("/", 1)[0].split(":", 1)[0] return "" @@ -112,4 +114,6 @@ def test_s90_locked_agents_cannot_resolve_or_reach_dns(): assert not out["random_resolves"] and not out["github_resolves"], f"{where}: outside names resolve: {out}" for key in ("udp_a", "udp_txt", "tcp_a"): assert not out[key]["answered"], f"{where}: kube-dns answered {key}: {out[key]}" - assert out["gateway"], f"{where}: positive control failed, the gateway name does not resolve from /etc/hosts" + host = _gateway_host(pod) + assert host, f"{where}: no gateway URL in the pod env; the positive control cannot run" + assert out["gateway"], f"{where}: positive control failed, {host} does not resolve from /etc/hosts" From 34ba46f9c303b880df228f559397e50060000283 Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 16:28:40 +0100 Subject: [PATCH 09/10] fix(platform): log CE-21 upstream and host-map fields in JSON lines The JSON formatter emits only allowlisted context keys, so the sidecar's upstream_unreachable line on tes2 showed the event but not which upstream, IP or fix hint. Allow upstream, host, ip, hint, configmap, hostnames and missing (names and addresses, never credentials); the sidecar reports the exception class under the existing exc_type key. Co-Authored-By: Claude Opus 5.5 --- images/common/zelkor_logging.py | 8 ++++++++ images/sidecar/zelkor_sidecar/egress.py | 2 +- tests/test_sidecar_upstream_probe.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/images/common/zelkor_logging.py b/images/common/zelkor_logging.py index 778d18e..d973a8a 100644 --- a/images/common/zelkor_logging.py +++ b/images/common/zelkor_logging.py @@ -45,6 +45,14 @@ "deny_reason", "request_bytes", "response_bytes", + # CE-21 agent host map: which upstream / name, never a credential. + "upstream", + "host", + "ip", + "hint", + "configmap", + "hostnames", + "missing", ) diff --git a/images/sidecar/zelkor_sidecar/egress.py b/images/sidecar/zelkor_sidecar/egress.py index 56f5712..27148d1 100644 --- a/images/sidecar/zelkor_sidecar/egress.py +++ b/images/sidecar/zelkor_sidecar/egress.py @@ -110,7 +110,7 @@ async def _reachable(self, name: str, url: str) -> bool: self._down.add(name) logger.error("upstream unreachable", extra={ "event": "upstream_unreachable", "upstream": name, "host": host, "ip": ip or "unresolved", - "error": type(exc).__name__, + "exc_type": type(exc).__name__, "hint": "host entries may be stale: run zelkor deploy (or delete the pod on K8s >= 1.36)"}) return False if name in self._down: diff --git a/tests/test_sidecar_upstream_probe.py b/tests/test_sidecar_upstream_probe.py index f3405e9..a4084e4 100644 --- a/tests/test_sidecar_upstream_probe.py +++ b/tests/test_sidecar_upstream_probe.py @@ -78,3 +78,18 @@ async def no_name(host, port): with caplog.at_level(logging.ERROR, logger="zelkor-sidecar"): assert asyncio.run(probe.check()) is False assert {r.ip for r in caplog.records} == {"unresolved"} + + +def test_json_log_line_carries_upstream_ip_and_hint(caplog): + sys.path.insert(0, str(ROOT / "images" / "common")) + import json as _json + from zelkor_logging import JsonFormatter + + net, now = Net(), [0.0] + net.down.add(80) + with caplog.at_level(logging.ERROR, logger="zelkor-sidecar"): + asyncio.run(_probe(net, now).check()) + line = _json.loads(JsonFormatter().format(caplog.records[0])) + assert line["event"] == "upstream_unreachable" + assert line["upstream"] == "llm" and line["ip"] == "10.43.175.236" and "zelkor deploy" in line["hint"] + From 0aee0e1dd70bc98e10112bc626fa2414f386fbef Mon Sep 17 00:00:00 2001 From: Alexander Vasilenko Date: Wed, 18 Nov 2026 17:07:14 +0100 Subject: [PATCH 10/10] fix(helm): read security.dnsLockdown nil-safe so --reuse-values upgrades render helm upgrade --reuse-values from a release that predates CE-21 carries no security.dnsLockdown (new chart defaults are not merged), and the templates failed on a nil value (demohetzner 2.5.5 upgrade). Two helpers read it with lockdown defaulting on and the writer deadline to 900 s. Co-Authored-By: Claude Opus 5.5 --- charts/zelkor-platform/templates/_helpers.tpl | 13 +++++++++++++ .../templates/contract-configmap.yaml | 2 +- .../templates/security/agent-hosts.yaml | 6 +++--- .../templates/security/networkpolicies.yaml | 6 +++--- tests/test_helm_dns_lockdown.py | 7 +++++++ 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/charts/zelkor-platform/templates/_helpers.tpl b/charts/zelkor-platform/templates/_helpers.tpl index 460b1f5..b75490b 100644 --- a/charts/zelkor-platform/templates/_helpers.tpl +++ b/charts/zelkor-platform/templates/_helpers.tpl @@ -1773,3 +1773,16 @@ Usage: include "zelkor-platform.dataToolEgress" (dict "egressTo" list "component {{- end -}} {{- toYaml $rules -}} {{- end }} + +{{/* +CE-21 agent DNS lockdown, nil-safe: `helm upgrade --reuse-values` from a release that predates the +key carries no security.dnsLockdown, and the lockdown must still default on. +*/}} +{{- define "zelkor-platform.dnsLockdownEnabled" -}} +{{- $dl := (.Values.security | default dict).dnsLockdown | default dict -}} +{{- if hasKey $dl "enabled" -}}{{- ternary "true" "false" (eq (toString $dl.enabled) "true") -}}{{- else -}}true{{- end -}} +{{- end }} + +{{- define "zelkor-platform.dnsLockdownWriterDeadline" -}} +{{- (((.Values.security | default dict).dnsLockdown | default dict).writerDeadlineSeconds | default 900) | int -}} +{{- end }} diff --git a/charts/zelkor-platform/templates/contract-configmap.yaml b/charts/zelkor-platform/templates/contract-configmap.yaml index 74a0167..f96833c 100644 --- a/charts/zelkor-platform/templates/contract-configmap.yaml +++ b/charts/zelkor-platform/templates/contract-configmap.yaml @@ -37,7 +37,7 @@ data: {{- /* CE-15a-4: HTTP APIs relayed as {release}-api-{name}. */}} HTTP_APIS: {{ join "," $apis | quote }} {{- end }} - {{- if .Values.security.dnsLockdown.enabled }} + {{- if eq (include "zelkor-platform.dnsLockdownEnabled" .) "true" }} {{- /* CE-21: agent pods get no DNS; their hostAliases come from this host map. */}} AGENT_HOSTS_CONFIGMAP: {{ printf "%s-agent-hosts" $full | quote }} {{- end }} diff --git a/charts/zelkor-platform/templates/security/agent-hosts.yaml b/charts/zelkor-platform/templates/security/agent-hosts.yaml index fb06f52..107f2ca 100644 --- a/charts/zelkor-platform/templates/security/agent-hosts.yaml +++ b/charts/zelkor-platform/templates/security/agent-hosts.yaml @@ -2,7 +2,7 @@ {{- /* CE-21: agent pods run without DNS. This writes the host map (one key per hostname, value = ClusterIP) that fills their hostAliases: the admission policy below on K8s >= 1.36, or `zelkor deploy` on older clusters. Plain Job, not a hook: Helm must not wait on it. */ -}} -{{- if .Values.security.dnsLockdown.enabled }} +{{- if eq (include "zelkor-platform.dnsLockdownEnabled" .) "true" }} {{- $fullname := include "zelkor-platform.fullname" . }} {{- $ns := .Release.Namespace }} {{- $cm := printf "%s-agent-hosts" $fullname }} @@ -108,7 +108,7 @@ metadata: spec: ttlSecondsAfterFinished: 600 backoffLimit: 10 - activeDeadlineSeconds: {{ int .Values.security.dnsLockdown.writerDeadlineSeconds }} + activeDeadlineSeconds: {{ include "zelkor-platform.dnsLockdownWriterDeadline" . }} template: metadata: labels: @@ -135,7 +135,7 @@ spec: - name: AGENT_HOSTS_RELEASE value: {{ .Release.Name | quote }} - name: AGENT_HOSTS_DEADLINE_S - value: {{ int .Values.security.dnsLockdown.writerDeadlineSeconds | quote }} + value: {{ include "zelkor-platform.dnsLockdownWriterDeadline" . | quote }} {{- include "zelkor-platform.logEnv" (dict "root" . "name" "zelkor-agent-hosts") | nindent 12 }} resources: requests: {cpu: 10m, memory: 32Mi} diff --git a/charts/zelkor-platform/templates/security/networkpolicies.yaml b/charts/zelkor-platform/templates/security/networkpolicies.yaml index 1bbf2da..8d3fb13 100644 --- a/charts/zelkor-platform/templates/security/networkpolicies.yaml +++ b/charts/zelkor-platform/templates/security/networkpolicies.yaml @@ -18,7 +18,7 @@ spec: policyTypes: - Egress egress: - {{- if not .Values.security.dnsLockdown.enabled }} + {{- if ne (include "zelkor-platform.dnsLockdownEnabled" .) "true" }} - to: - namespaceSelector: matchLabels: @@ -69,7 +69,7 @@ spec: port: 3000 {{- end }} {{- if .Values.aegra.enabled }} -{{- if .Values.security.dnsLockdown.enabled }} +{{- if eq (include "zelkor-platform.dnsLockdownEnabled" .) "true" }} --- # CE-21: the platform Aegra and Aegra-labelled pods that are not locked down keep DNS. A locked # agent with Valkey access matches aegra-egress too, so port 53 cannot live in that policy. @@ -153,7 +153,7 @@ spec: policyTypes: - Egress egress: - {{- if not .Values.security.dnsLockdown.enabled }} + {{- if ne (include "zelkor-platform.dnsLockdownEnabled" .) "true" }} - to: - namespaceSelector: matchLabels: diff --git a/tests/test_helm_dns_lockdown.py b/tests/test_helm_dns_lockdown.py index 473dbac..7d6a9a7 100644 --- a/tests/test_helm_dns_lockdown.py +++ b/tests/test_helm_dns_lockdown.py @@ -112,3 +112,10 @@ def test_lockdown_off_keeps_dns_and_renders_no_writer(): assert not [j for j in _kind(docs, "Job") if "agent-hosts" in j["metadata"]["name"]] assert not _kind(docs, "MutatingAdmissionPolicy") assert _allows_dns(_agent_egress(docs)) + + +def test_reuse_values_upgrade_without_the_key_still_locks_down(): + """`helm upgrade --reuse-values` from a pre-CE-21 release carries no security.dnsLockdown.""" + docs = _docs("--set", "security.dnsLockdown=null") + assert _writer_env(docs)["AGENT_HOSTS_DEADLINE_S"] == "900" + assert not _allows_dns(_agent_egress(docs))