ROSAENG-726: Spike request-serving isolation on EKS (kata + qemu)

.gitignore

@@ -45,3 +45,4 @@ ephemeral-logs*
 
 # Dashboard (generated locally by ./dashboard/fetch-data.sh)
 dashboard/data.json
+.claude/worktrees/

argocd/config/management-cluster/hypershift/values.yaml

@@ -20,4 +20,4 @@ hypershift:
     # - aws-iam-auth-config ConfigMap sync from HC to HCP namespace (HC controller)
     # - hypershift.openshift.io/aws-iam-authenticator annotation propagation
     # Branch: https://github.com/typeid/hypershift/tree/aws-iam-authenticator-poc
-    image: quay.io/cbusse_openshift/hypershift-operator:aws-iam-auth
+    image: quay.io/cbusse_openshift/hypershift-operator:kata-poc

argocd/config/management-cluster/kata-deploy/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v2
+name: kata-deploy
+description: Kata Containers QEMU runtime for request-serving pod isolation
+type: application
+version: 0.4.0
+dependencies:
+  - name: kata-deploy
+    version: "4.1.0"
+    repository: "oci://ghcr.io/kata-containers/kata-deploy-charts"

argocd/config/management-cluster/kata-deploy/templates/runtimeclass.yaml

@@ -0,0 +1,16 @@
+# RuntimeClass for kata-powered request-serving pods.
+#
+# NAME "kata-fc": HyperShift hardcodes runtimeClassName=kata-fc on
+# request-serving pods. The name is just a label.
+#
+# HANDLER "kata-qemu": Upstream kata-deploy registers this handler in
+# CRI-O, pointing to the kata shim which uses QEMU.
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: {{ .Values.runtimeClass.name }}
+handler: {{ .Values.runtimeClass.handler }}
+overhead:
+  podFixed:
+    cpu: {{ .Values.runtimeClass.overhead.cpu | quote }}
+    memory: {{ .Values.runtimeClass.overhead.memory | quote }}

argocd/config/management-cluster/kata-deploy/values.yaml

@@ -0,0 +1,30 @@
+# Upstream kata-deploy subchart configuration.
+# Not FIPS-compliant — for spike only.
+kata-deploy:
+  k8sDistribution: "k8s"
+  deploymentMode: daemonset
+  defaultShim:
+    amd64: qemu
+  shims:
+    disableAll: true
+    qemu:
+      enabled: true
+  runtimeClasses:
+    enabled: false
+  node-feature-discovery:
+    enabled: false
+  nodeSelector:
+    karpenter.sh/nodepool: kata-request-serving
+  tolerations:
+    - key: hypershift.openshift.io/request-serving-component
+      operator: Exists
+      effect: NoSchedule
+
+# Our RuntimeClass (managed separately from the subchart).
+# HyperShift hardcodes runtimeClassName=kata-fc on request-serving pods.
+runtimeClass:
+  name: kata-fc
+  handler: kata-qemu
+  overhead:
+    cpu: "250m"
+    memory: "350Mi"

argocd/config/management-cluster/kata-nodepool/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: kata-nodepool
+description: Karpenter NodePool and EC2NodeClass for Kata Containers request-serving nodes
+type: application
+version: 0.1.0

argocd/config/management-cluster/kata-nodepool/templates/00-nodeclass.yaml

@@ -0,0 +1,33 @@
+{{- $clusterName := required "global.cluster_name must be set via ApplicationSet valuesObject" .Values.global.cluster_name -}}
+apiVersion: karpenter.k8s.aws/v1
+kind: EC2NodeClass
+metadata:
+  name: kata
+spec:
+  amiFamily: AL2023
+  amiSelectorTerms:
+    - alias: al2023@latest
+  instanceProfile: {{ $clusterName }}-karpenter-node-role
+  subnetSelectorTerms:
+    - tags:
+        kubernetes.io/cluster/{{ $clusterName }}: owned
+  securityGroupSelectorTerms:
+    - tags:
+        aws:eks:cluster-name: {{ $clusterName }}
+  metadataOptions:
+    httpTokens: required
+    httpPutResponseHopLimit: 2
+  cpuOptions:
+    nestedVirtualization: "enabled"
+  blockDeviceMappings:
+    - deviceName: /dev/xvda
+      ebs:
+        volumeSize: {{ .Values.kataNodePool.rootVolumeSize | default "100Gi" }}
+        volumeType: gp3
+        encrypted: true
+        deleteOnTermination: true
+  userData: |
+    #!/bin/bash
+    set -euo pipefail
+    modprobe kvm_intel 2>/dev/null || true
+    echo "kvm_intel" > /etc/modules-load.d/kvm.conf

argocd/config/management-cluster/kata-nodepool/templates/10-nodepool.yaml

@@ -0,0 +1,44 @@
+apiVersion: karpenter.sh/v1
+kind: NodePool
+metadata:
+  name: kata-request-serving
+spec:
+  template:
+    metadata:
+      labels:
+        katacontainers.io/kata-runtime: "true"
+    spec:
+      nodeClassRef:
+        group: karpenter.k8s.aws
+        kind: EC2NodeClass
+        name: kata
+      taints:
+        - key: hypershift.openshift.io/request-serving-component
+          value: "true"
+          effect: NoSchedule
+      requirements:
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values:
+            - {{ .Values.kataNodePool.capacityType }}
+        - key: kubernetes.io/arch
+          operator: In
+          values:
+            - {{ .Values.kataNodePool.arch }}
+        - key: node.kubernetes.io/instance-type
+          operator: In
+          values:
+            {{- range .Values.kataNodePool.instanceFamilies }}
+            {{- $family := . }}
+            {{- range list "xlarge" "2xlarge" "4xlarge" }}
+            - {{ $family }}.{{ . }}
+            {{- end }}
+            {{- end }}
+  limits:
+    cpu: {{ .Values.kataNodePool.limits.cpu | quote }}
+    memory: {{ .Values.kataNodePool.limits.memory }}
+  disruption:
+    consolidationPolicy: {{ .Values.kataNodePool.disruption.consolidationPolicy }}
+    consolidateAfter: {{ .Values.kataNodePool.disruption.consolidateAfter }}
+  # TODO: add topology spread constraints for AZ balancing
+  # TODO: tune limits based on observed HCP density

argocd/config/management-cluster/kata-nodepool/values.yaml

@@ -0,0 +1,15 @@
+kataNodePool:
+  instanceFamilies:
+    - m7i
+    - c7i
+  capacityType: on-demand
+  arch: amd64
+  limits:
+    cpu: "64"
+    memory: 256Gi
+  disruption:
+    consolidationPolicy: WhenEmptyOrUnderutilized
+    consolidateAfter: 30s
+
+global:
+  cluster_name: ""

argocd/config/regional-cluster/hyperfleet/values.yaml

@@ -9,7 +9,7 @@ hyperfleet:
 
   source:
     repoURL: https://github.com/openshift-online/rosa-hyperfleet-api.git
-    targetRevision: dab88f7dd47f239c3318043a9d849987b0ac5d70
+    targetRevision: main
     path: hyperfleet-operator/charts
 
   syncPolicy:
@@ -19,8 +19,8 @@ hyperfleet:
 
   helmValues:
     image:
-      repository: quay.io/redhat-user-workloads/rosa-tenant/hyperfleet-operator
-      tag: "dab88f7dd47f239c3318043a9d849987b0ac5d70"
+      repository: quay.io/cbusse_openshift/hyperfleet-operator
+      tag: "kata-poc"
       pullPolicy: Always
 
     replicaCount: 2