diff --git a/.gitignore b/.gitignore index b369e2e41..38b1129fb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ aaq_controller aaq_operator aaq_server tmp -_ci-configs/* +kubevirtci/_ci-configs/* 0 bin/ _out/* diff --git a/Makefile b/Makefile index 3bd3ec30a..aaec9d42a 100755 --- a/Makefile +++ b/Makefile @@ -23,7 +23,8 @@ goveralls \ release-description \ bazel-build-images push-images \ - fossa + fossa \ + bump-kubevirtci all: build build: aaq_controller aaq_server aaq_operator @@ -39,10 +40,6 @@ else DO_BAZ=eval endif -ifeq ($(origin KUBEVIRT_RELEASE), undefined) - KUBEVIRT_RELEASE="latest_nightly" -endif - all: manifests build-images manifests: @@ -59,10 +56,10 @@ generate-verify: generate ./hack/check-for-binaries.sh cluster-up: - eval "KUBEVIRT_RELEASE=${KUBEVIRT_RELEASE} ./cluster-up/up.sh" + ./hack/cluster-up.sh cluster-down: - ./cluster-up/down.sh + ./kubevirtci/cluster-up/down.sh push-images: eval "DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} ./hack/build/build-docker.sh push" @@ -122,3 +119,6 @@ fmt: run: build sudo ./aaq_controller + +bump-kubevirtci: + ./hack/bump-kubevirtci.sh diff --git a/README.md b/README.md index 976f7b94d..2dc1eefe6 100644 --- a/README.md +++ b/README.md @@ -224,14 +224,14 @@ Deploying the AAQ controller is straightforward. ### Deploy it with our CI system -AAQ includes a self-contained development and test environment. We use Docker to build, and we provide a simple way to get a test cluster up and running on your laptop. The development tools include a version of kubectl that you can use to communicate with the cluster. A wrapper script to communicate with the cluster can be invoked using ./cluster-up/kubectl.sh. +AAQ includes a self-contained development and test environment. We use Docker to build, and we provide a simple way to get a test cluster up and running on your laptop. The development tools include a version of kubectl that you can use to communicate with the cluster. A wrapper script to communicate with the cluster can be invoked using ./kubevirtci/cluster-up/kubectl.sh. ```bash $ mkdir $GOPATH/src/kubevirt.io && cd $GOPATH/src/kubevirt.io $ git clone https://github.com/kubevirt/application-aware-quota && cd application-aware-quota $ make cluster-up $ make cluster-sync -$ ./cluster-up/kubectl.sh ..... +$ ./kubevirtci/cluster-up/kubectl.sh ..... ``` For development on external cluster (not provisioned by our AAQ), check out the [external provider](cluster-sync/external/README.md). diff --git a/automation/test.sh b/automation/test.sh index dc232a83c..2d33ecb92 100755 --- a/automation/test.sh +++ b/automation/test.sh @@ -46,14 +46,14 @@ elif [[ $TARGET =~ k8s-.* ]]; then export KUBEVIRT_MEMORY_SIZE=8192 fi -if [ ! -d "cluster-up/cluster/$KUBEVIRT_PROVIDER" ]; then +if [ ! -d "kubevirtci/cluster-up/cluster/$KUBEVIRT_PROVIDER" ]; then echo "The cluster provider $KUBEVIRT_PROVIDER does not exist" exit 1 fi export KUBEVIRT_NUM_NODES=2 -kubectl() { cluster-up/kubectl.sh "$@"; } +kubectl() { kubevirtci/cluster-up/kubectl.sh "$@"; } export AAQ_NAMESPACE="${AAQ_NAMESPACE:-aaq}" diff --git a/cluster-sync/clean.sh b/cluster-sync/clean.sh index 45af85338..5c3273219 100755 --- a/cluster-sync/clean.sh +++ b/cluster-sync/clean.sh @@ -1,8 +1,9 @@ #!/bin/bash -e source ./hack/build/config.sh -source ./cluster-up/hack/common.sh -source ./cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh +source ./hack/config-kubevirtci.sh +source ./kubevirtci/cluster-up/hack/common.sh +source ./kubevirtci/cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh echo "Cleaning up ..." diff --git a/cluster-sync/install.sh b/cluster-sync/install.sh index da6bde745..81be55f65 100644 --- a/cluster-sync/install.sh +++ b/cluster-sync/install.sh @@ -22,4 +22,3 @@ function wait_aaq_crd_installed { exit 1 fi } - diff --git a/cluster-sync/sync.sh b/cluster-sync/sync.sh index 12a3d636c..5988a42e6 100755 --- a/cluster-sync/sync.sh +++ b/cluster-sync/sync.sh @@ -7,8 +7,8 @@ echo aaq source ./hack/build/config.sh source ./hack/build/common.sh -source ./cluster-up/hack/common.sh -source ./cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh +source ./kubevirtci/cluster-up/hack/common.sh +source ./kubevirtci/cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh if [ "${KUBEVIRT_PROVIDER}" = "external" ]; then AAQ_SYNC_PROVIDER="external" diff --git a/cluster-up/cluster/k8s-1.30/provider.sh b/cluster-up/cluster/k8s-1.30/provider.sh deleted file mode 100644 index e2bf40cda..000000000 --- a/cluster-up/cluster/k8s-1.30/provider.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -e - -if [ "${KUBEVIRT_CGROUPV2}" == "true" ]; then - export KUBEVIRT_PROVIDER_EXTRA_ARGS="${KUBEVIRT_PROVIDER_EXTRA_ARGS} --kernel-args='systemd.unified_cgroup_hierarchy=1'" -fi - -# shellcheck disable=SC1090 -source "${KUBEVIRTCI_PATH}/cluster/k8s-provider-common.sh" diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh deleted file mode 100644 index c70a4bf8d..000000000 --- a/cluster-up/cluster/k8s-provider-common.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# shellcheck source=cluster-up/cluster/ephemeral-provider-common.sh -source "${KUBEVIRTCI_PATH}/cluster/ephemeral-provider-common.sh" - - -#if UNLIMITEDSWAP is set to true - Kubernetes workloads can use as much swap memory as they request, up to the system limit. -#otherwise Kubernetes workloads can use as much swap memory as they request, up to the system limit by default -function configure_swap_memory () { - if [ "$KUBEVIRT_SWAP_ON" == "true" ] ;then - for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do - if [ ! -z $KUBEVIRT_SWAP_SIZE_IN_GB ]; then - $ssh node${nodeNum} -- sudo dd if=/dev/zero of=/swapfile count=$KUBEVIRT_SWAP_SIZE_IN_GB bs=1G - $ssh node${nodeNum} -- sudo mkswap /swapfile - fi - - $ssh node${nodeNum} -- sudo swapon -a - - if [ ! -z $KUBEVIRT_SWAPPINESS ]; then - $ssh node${nodeNum} -- "sudo /bin/su -c \"echo vm.swappiness = $KUBEVIRT_SWAPPINESS >> /etc/sysctl.conf\"" - $ssh node${nodeNum} -- sudo sysctl vm.swappiness=$KUBEVIRT_SWAPPINESS - fi - - if [ $KUBEVIRT_UNLIMITEDSWAP == "true" ]; then - $ssh node${nodeNum} -- "sudo sed -i ':a;N;\$!ba;s/memorySwap: {}/memorySwap:\n swapBehavior: UnlimitedSwap/g' /var/lib/kubelet/config.yaml" - $ssh node${nodeNum} -- sudo systemctl restart kubelet - fi - done - fi -} - -function configure_ksm_module () { - if [ "$KUBEVIRT_KSM_ON" == "true" ] ;then - for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do - $ssh node${nodeNum} -- "echo 1 | sudo tee /sys/kernel/mm/ksm/run >/dev/null" - if [ ! -z $KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS ]; then - $ssh node${nodeNum} -- "echo ${KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS} | sudo tee /sys/kernel/mm/ksm/sleep_millisecs >/dev/null " - fi - if [ ! -z $KUBEVIRT_KSM_PAGES_TO_SCAN ]; then - $ssh node${nodeNum} -- "echo ${KUBEVIRT_KSM_PAGES_TO_SCAN} | sudo tee /sys/kernel/mm/ksm/pages_to_scan >/dev/null " - fi - done - fi -} - -function configure_memory_overcommitment_behavior () { - configure_swap_memory - configure_ksm_module -} - -function deploy_cnao() { - if [ "$KUBEVIRT_WITH_CNAO" == "true" ] || [ "$KUBVIRT_WITH_CNAO_SKIP_CONFIG" == "true" ]; then - $kubectl create -f /opt/cnao/namespace.yaml - $kubectl create -f /opt/cnao/network-addons-config.crd.yaml - $kubectl create -f /opt/cnao/operator.yaml - - if [ "$KUBVIRT_WITH_CNAO_SKIP_CONFIG" != "true" ]; then - create_network_addons_config - fi - - # Install whereabouts on CNAO lanes - $kubectl create -f /opt/whereabouts - fi -} - -function create_network_addons_config() { - local nac="/opt/cnao/network-addons-config-example.cr.yaml" - if [ "$KUBEVIRT_WITH_MULTUS_V3" == "true" ]; then - local no_multus_nac="/opt/cnao/no-multus-nac.yaml" - $ssh node01 -- "awk '!/multus/' ${nac} | sudo tee ${no_multus_nac}" - nac=${no_multus_nac} - fi - - $kubectl apply -f ${nac} -} - -function wait_for_cnao_ready() { - if [ "$KUBEVIRT_WITH_CNAO" == "true" ] || [ "$KUBVIRT_WITH_CNAO_SKIP_CONFIG" == "true" ]; then - $kubectl wait deployment -n cluster-network-addons cluster-network-addons-operator --for condition=Available --timeout=200s - if [ "$KUBVIRT_WITH_CNAO_SKIP_CONFIG" != "true" ]; then - $kubectl wait networkaddonsconfig cluster --for condition=Available --timeout=200s - fi - fi -} - -function deploy_multus() { - if [ "$KUBEVIRT_WITH_MULTUS_V3" == "true" ]; then - $kubectl create -f /opt/multus/multus.yaml - fi -} - -function wait_for_multus_ready() { - if [ "$KUBEVIRT_WITH_MULTUS_V3" == "true" ]; then - $kubectl rollout status -n kube-system ds/kube-multus-ds --timeout=200s - fi -} - -function deploy_istio() { - if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ]; then - if [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then - $kubectl create -f /opt/istio/istio-operator-with-cnao.cr.yaml - else - $kubectl create -f /opt/istio/istio-operator.cr.yaml - fi - fi -} - -function wait_for_istio_ready() { - if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ]; then - istio_operator_ns=istio-system - retries=0 - max_retries=20 - while [[ $retries -lt $max_retries ]]; do - echo "waiting for istio-operator to be healthy" - sleep 5 - health=$($kubectl -n $istio_operator_ns get istiooperator istio-operator -o jsonpath="{.status.status}") - if [[ $health == "HEALTHY" ]]; then - break - fi - retries=$((retries + 1)) - done - if [ $retries == $max_retries ]; then - echo "waiting istio-operator to be healthy failed" - exit 1 - fi - fi -} - -# copy_istio_cni_conf_files copy the generated Istio CNI net conf file -# (at '/etc/cni/multus/net.d/') to where Multus expect CNI net conf files ('/etc/cni/net.d/') -function copy_istio_cni_conf_files() { - if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ] && [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then - for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do - $ssh node${nodeNum} -- sudo cp -uv /etc/cni/multus/net.d/*istio*.conf /etc/cni/net.d/ - done - fi -} - -function deploy_cdi() { - if [ "$KUBEVIRT_DEPLOY_CDI" == "true" ]; then - if [ -n "${KUBEVIRT_CUSTOM_CDI_VERSION}" ]; then - $ssh node01 -- 'sudo sed --regexp-extended -i s/v[0-9]+\.[0-9]+\.[0-9]+\(.*\)?$/'"$KUBEVIRT_CUSTOM_CDI_VERSION"'/g /opt/cdi-*-operator.yaml' - fi - - $kubectl create -f /opt/cdi-*-operator.yaml - $kubectl create -f /opt/cdi-*-cr.yaml - fi -} - -function wait_for_cdi_ready() { - if [ "$KUBEVIRT_DEPLOY_CDI" == "true" ]; then - while [ "$($kubectl get pods --namespace cdi | grep -c 'cdi-')" -lt 4 ]; do - $kubectl get pods --namespace cdi - sleep 10 - done - $kubectl wait --for=condition=Ready pod --timeout=180s --all --namespace cdi - fi -} - -# configure Prometheus to select kubevirt prometheusrules -function configure_prometheus() { - if [[ $KUBEVIRT_DEPLOY_PROMETHEUS == "true" ]] && $kubectl get crd prometheuses.monitoring.coreos.com; then - _kubectl patch prometheus k8s -n monitoring --type='json' -p='[{"op": "replace", "path": "/spec/ruleSelector", "value":{}}, {"op": "replace", "path": "/spec/ruleNamespaceSelector", "value":{"matchLabels": {}}}]' - fi -} - -function up() { - params=$(_add_common_params) - if echo "$params" | grep -q ERROR; then - echo -e "$params" - exit 1 - fi - eval ${_cli:?} run $params - - ${_cli} scp --prefix $provider_prefix /etc/kubernetes/admin.conf - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig - - # Set server and disable tls check - export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig - kubectl config set-cluster kubernetes --server="https://$(_main_ip):$(_port k8s)" - kubectl config set-cluster kubernetes --insecure-skip-tls-verify=true - - # Workaround https://github.com/containers/conmon/issues/315 by not dumping the file to stdout for the time being - if [[ ${_cri_bin} = podman* ]]; then - k8s_version=$(kubectl get node node01 --no-headers -o=custom-columns=VERSION:.status.nodeInfo.kubeletVersion) - curl -Ls "https://dl.k8s.io/release/${k8s_version}/bin/linux/amd64/kubectl" -o ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl - else - ${_cli} scp --prefix ${provider_prefix:?} /usr/bin/kubectl - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl - fi - - chmod u+x ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl - - # Make sure that local config is correct - prepare_config - ssh="${_cli} --prefix $provider_prefix ssh" - kubectl="$ssh node01 -- sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf" - - # For multinode cluster Label all the non control-plane nodes as workers, - # for one node cluster label control-plane with 'control-plane,worker' roles - if [ "$KUBEVIRT_NUM_NODES" -gt 1 ]; then - label="!node-role.kubernetes.io/control-plane" - else - label="node-role.kubernetes.io/control-plane" - fi - $kubectl label node -l $label node-role.kubernetes.io/worker='' - - configure_prometheus - configure_memory_overcommitment_behavior - - deploy_cnao - deploy_multus - deploy_istio - deploy_cdi - - until wait_for_cnao_ready && wait_for_istio_ready && wait_for_cdi_ready && wait_for_multus_ready; do - echo "Waiting for cluster components..." - sleep 5 - done - - # FIXME: remove 'copy_istio_cni_conf_files()' as soon as [1] and [2] are resolved - # [1] https://github.com/kubevirt/kubevirtci/issues/906 - # [2] https://github.com/k8snetworkplumbingwg/multus-cni/issues/982 - copy_istio_cni_conf_files -} diff --git a/cluster-up/cluster/kind-1.27-vgpu/image b/cluster-up/cluster/kind-1.27-vgpu/image deleted file mode 100644 index cadea9ed8..000000000 --- a/cluster-up/cluster/kind-1.27-vgpu/image +++ /dev/null @@ -1 +0,0 @@ -kindest/node:v1.27.1@sha256:9915f5629ef4d29f35b478e819249e89cfaffcbfeebda4324e5c01d53d937b09 diff --git a/cluster-up/cluster/kind-1.27-vgpu/version b/cluster-up/cluster/kind-1.27-vgpu/version deleted file mode 100644 index 66333910a..000000000 --- a/cluster-up/cluster/kind-1.27-vgpu/version +++ /dev/null @@ -1 +0,0 @@ -0.18.0 diff --git a/cluster-up/cluster/kind-sriov/OWNERS b/cluster-up/cluster/kind-sriov/OWNERS deleted file mode 100644 index 786c313b2..000000000 --- a/cluster-up/cluster/kind-sriov/OWNERS +++ /dev/null @@ -1,10 +0,0 @@ -filters: - ".*": - reviewers: - - qinqon - - oshoval - - phoracek - - ormergi - approvers: - - qinqon - - phoracek diff --git a/cluster-up/cluster/kind-sriov/image b/cluster-up/cluster/kind-sriov/image deleted file mode 100644 index 4507c4604..000000000 --- a/cluster-up/cluster/kind-sriov/image +++ /dev/null @@ -1 +0,0 @@ -kindest/node:v1.28.0@sha256:dad5a6238c5e41d7cac405fae3b5eda2ad1de6f1190fa8bfc64ff5bb86173213 diff --git a/cluster-up/cluster/kind-sriov/sriov-components/manifests/patch-node-selector.yaml.in b/cluster-up/cluster/kind-sriov/sriov-components/manifests/patch-node-selector.yaml.in deleted file mode 100644 index 0117c8cdd..000000000 --- a/cluster-up/cluster/kind-sriov/sriov-components/manifests/patch-node-selector.yaml.in +++ /dev/null @@ -1,3 +0,0 @@ -- op: add - path: /spec/template/spec/nodeSelector/$LABEL_KEY - value: "$LABEL_VALUE" diff --git a/cluster-up/cluster/kind-sriov/sriov-components/manifests/sriovdp-daemonset.yaml b/cluster-up/cluster/kind-sriov/sriov-components/manifests/sriovdp-daemonset.yaml deleted file mode 100644 index 322a22397..000000000 --- a/cluster-up/cluster/kind-sriov/sriov-components/manifests/sriovdp-daemonset.yaml +++ /dev/null @@ -1,221 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: sriov-device-plugin - namespace: kube-system - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-sriov-device-plugin-amd64 - namespace: kube-system - labels: - tier: node - app: sriovdp -spec: - selector: - matchLabels: - name: sriov-device-plugin - template: - metadata: - labels: - name: sriov-device-plugin - tier: node - app: sriovdp - spec: - hostNetwork: true - nodeSelector: - beta.kubernetes.io/arch: amd64 - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - serviceAccountName: sriov-device-plugin - containers: - - name: kube-sriovdp - image: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.4.0 - imagePullPolicy: IfNotPresent - args: - - --log-dir=sriovdp - - --log-level=10 - securityContext: - privileged: true - resources: - requests: - cpu: "250m" - memory: "40Mi" - limits: - cpu: 1 - memory: "200Mi" - volumeMounts: - - name: devicesock - mountPath: /var/lib/kubelet/ - readOnly: false - - name: log - mountPath: /var/log - - name: config-volume - mountPath: /etc/pcidp - - name: device-info - mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp - volumes: - - name: devicesock - hostPath: - path: /var/lib/kubelet/ - - name: log - hostPath: - path: /var/log - - name: device-info - hostPath: - path: /var/run/k8s.cni.cncf.io/devinfo/dp - type: DirectoryOrCreate - - name: config-volume - configMap: - name: sriovdp-config - items: - - key: config.json - path: config.json - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-sriov-device-plugin-ppc64le - namespace: kube-system - labels: - tier: node - app: sriovdp -spec: - selector: - matchLabels: - name: sriov-device-plugin - template: - metadata: - labels: - name: sriov-device-plugin - tier: node - app: sriovdp - spec: - hostNetwork: true - nodeSelector: - beta.kubernetes.io/arch: ppc64le - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - serviceAccountName: sriov-device-plugin - containers: - - name: kube-sriovdp - image: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:latest-ppc64le - imagePullPolicy: IfNotPresent - args: - - --log-dir=sriovdp - - --log-level=10 - securityContext: - privileged: true - resources: - requests: - cpu: "250m" - memory: "40Mi" - limits: - cpu: 1 - memory: "200Mi" - volumeMounts: - - name: devicesock - mountPath: /var/lib/kubelet/ - readOnly: false - - name: log - mountPath: /var/log - - name: config-volume - mountPath: /etc/pcidp - - name: device-info - mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp - volumes: - - name: devicesock - hostPath: - path: /var/lib/kubelet/ - - name: log - hostPath: - path: /var/log - - name: device-info - hostPath: - path: /var/run/k8s.cni.cncf.io/devinfo/dp - type: DirectoryOrCreate - - name: config-volume - configMap: - name: sriovdp-config - items: - - key: config.json - path: config.json ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-sriov-device-plugin-arm64 - namespace: kube-system - labels: - tier: node - app: sriovdp -spec: - selector: - matchLabels: - name: sriov-device-plugin - template: - metadata: - labels: - name: sriov-device-plugin - tier: node - app: sriovdp - spec: - hostNetwork: true - nodeSelector: - beta.kubernetes.io/arch: arm64 - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - serviceAccountName: sriov-device-plugin - containers: - - name: kube-sriovdp - image: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:latest-arm64 - imagePullPolicy: IfNotPresent - args: - - --log-dir=sriovdp - - --log-level=10 - securityContext: - privileged: true - resources: - requests: - cpu: "250m" - memory: "40Mi" - limits: - cpu: 1 - memory: "200Mi" - volumeMounts: - - name: devicesock - mountPath: /var/lib/kubelet/ - readOnly: false - - name: log - mountPath: /var/log - - name: config-volume - mountPath: /etc/pcidp - - name: device-info - mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp - volumes: - - name: devicesock - hostPath: - path: /var/lib/kubelet/ - - name: log - hostPath: - path: /var/log - - name: device-info - hostPath: - path: /var/run/k8s.cni.cncf.io/devinfo/dp - type: DirectoryOrCreate - - name: config-volume - configMap: - name: sriovdp-config - items: - - key: config.json - path: config.json diff --git a/cluster-up/cluster/kind-sriov/version b/cluster-up/cluster/kind-sriov/version deleted file mode 100644 index 1cf0537c3..000000000 --- a/cluster-up/cluster/kind-sriov/version +++ /dev/null @@ -1 +0,0 @@ -0.19.0 diff --git a/cluster-up/up.sh b/cluster-up/up.sh deleted file mode 100755 index ccdcd6290..000000000 --- a/cluster-up/up.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -function validate_single_stack_ipv6() { - local kube_ns="kube-system" - local pod_label="calico-kube-controllers" - - echo "validating provider is single stack IPv6" - until _kubectl wait --for=condition=Ready pod --timeout=10s -n $kube_ns -lk8s-app=${pod_label}; do sleep 1; done > /dev/null 2>&1 - - local pod=$(_kubectl get pods -n ${kube_ns} -lk8s-app=${pod_label} -o=custom-columns=NAME:.metadata.name --no-headers) - local primary_ip=$(_kubectl get pod -n ${kube_ns} ${pod} -ojsonpath="{ @.status.podIP }") - - if [[ ! ${primary_ip} =~ fd00 ]]; then - echo "error: single stack primary ip ($primary_ip) is not IPv6 as expected" - exit 1 - fi - - if _kubectl get pod -n ${kube_ns} ${pod} -ojsonpath="{ @.status.podIPs[1] }" > /dev/null 2>&1; then - echo "error: single stack cluster expected" - exit 1 - fi -} - -if [ -z "$KUBEVIRTCI_PATH" ]; then - KUBEVIRTCI_PATH="$( - cd "$(dirname "$BASH_SOURCE[0]")/" - echo "$(pwd)/" - )" -fi - - -source ${KUBEVIRTCI_PATH}hack/common.sh -source ${KUBEVIRTCI_CLUSTER_PATH}/$KUBEVIRT_PROVIDER/provider.sh -up - -# check if the environment has a corrupted host -if [[ $(${KUBEVIRTCI_PATH}kubectl.sh get nodes | grep localhost) != "" ]]; then - echo "The environment has a corrupted host" - exit 1 -fi - -if [ ${KUBEVIRT_SINGLE_STACK} == true ]; then - validate_single_stack_ipv6 -fi - -if [ "$KUBEVIRT_RELEASE" = "latest_nightly" ]; then - LATEST=$(curl -L https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/latest) - kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-operator.yaml - kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-cr.yaml -elif [ "$KUBEVIRT_RELEASE" = "latest_stable" ]; then - RELEASE=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt) - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-operator.yaml - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-cr.yaml -else - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-operator.yaml - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-cr.yaml -fi -# Ensure the KubeVirt CRD is created -count=0 -until kubectl get crd kubevirts.kubevirt.io; do - ((count++)) && ((count == 30)) && echo "KubeVirt CRD not found" && exit 1 - echo "waiting for KubeVirt CRD" - sleep 1 -done - -# Ensure the KubeVirt API is available -count=0 -until kubectl api-resources --api-group=kubevirt.io | grep kubevirts; do - ((count++)) && ((count == 30)) && echo "KubeVirt API not found" && exit 1 - echo "waiting for KubeVirt API" - sleep 1 -done - - -# Ensure the KubeVirt CR is created -count=0 -until kubectl -n kubevirt get kv kubevirt; do - ((count++)) && ((count == 30)) && echo "KubeVirt CR not found" && exit 1 - echo "waiting for KubeVirt CR" - sleep 1 -done - -# Wait until KubeVirt is ready -count=0 -until kubectl wait -n kubevirt kv kubevirt --for condition=Available --timeout 5m; do - ((count++)) && ((count == 5)) && echo "KubeVirt not ready in time" && exit 1 - echo "Error waiting for KubeVirt to be Available, sleeping 1m and retrying" - sleep 1m -done diff --git a/hack/build/bazel-docker.sh b/hack/build/bazel-docker.sh index 3510c4c7c..d35f99fa7 100755 --- a/hack/build/bazel-docker.sh +++ b/hack/build/bazel-docker.sh @@ -94,9 +94,9 @@ _rsync \ --exclude 'bazel-application-aware-quota' \ --exclude 'bazel-out' \ --exclude 'bazel-testlogs' \ - --exclude 'cluster-up/cluster/**/.kubectl' \ - --exclude 'cluster-up/cluster/**/.oc' \ - --exclude 'cluster-up/cluster/**/.kubeconfig' \ + --exclude 'kubevirtci/cluster-up/cluster/**/.kubectl' \ + --exclude 'kubevirtci/cluster-up/cluster/**/.oc' \ + --exclude 'kubevirtci/cluster-up/cluster/**/.kubeconfig' \ --exclude ".vagrant" \ ${AAQ_DIR}/ \ "rsync://root@127.0.0.1:${RSYNCD_PORT}/build" @@ -137,9 +137,9 @@ _rsync \ --exclude 'bazel-application-aware-quota' \ --exclude 'bazel-out' \ --exclude 'bazel-testlogs' \ - --exclude 'cluster-up/cluster/**/.kubectl' \ - --exclude 'cluster-up/cluster/**/.oc' \ - --exclude 'cluster-up/cluster/**/.kubeconfig' \ + --exclude 'kubevirtci/kubevirtci/cluster-up/cluster/**/.kubectl' \ + --exclude 'kubevirtci/kubevirtci/cluster-up/cluster/**/.oc' \ + --exclude 'kubevirtci/kubevirtci/cluster-up/cluster/**/.kubeconfig' \ --exclude "_out" \ --exclude "bin" \ --exclude "vendor" \ diff --git a/hack/build/common.sh b/hack/build/common.sh index 8401e35a2..aa2362c64 100755 --- a/hack/build/common.sh +++ b/hack/build/common.sh @@ -46,6 +46,7 @@ VENDOR_DIR=${AAQ_DIR}/vendor ARCHITECTURE="${BUILD_ARCH:-$(uname -m)}" HOST_ARCHITECTURE="$(uname -m)" AAQ_CRI="$(determine_aaq_bin)" + if [ "${AAQ_CRI}" = "docker" ]; then AAQ_CONTAINER_BUILDCMD=${AAQ_CONTAINER_BUILDCMD:-docker} else diff --git a/hack/build/run-functional-tests.sh b/hack/build/run-functional-tests.sh index 75052ed25..dcd8b4cc1 100755 --- a/hack/build/run-functional-tests.sh +++ b/hack/build/run-functional-tests.sh @@ -22,22 +22,22 @@ readonly AAQ_WAIT_TIME=10 script_dir="$(cd "$(dirname "$0")" && pwd -P)" source hack/build/config.sh source hack/build/common.sh -source cluster-up/hack/common.sh +source kubevirtci/cluster-up/hack/common.sh KUBEVIRTCI_CONFIG_PATH="$( cd "$(dirname "$BASH_SOURCE[0]")/../../" - echo "$(pwd)/_ci-configs" + echo "$(pwd)/kubevirtci/_ci-configs" )" # functional testing BASE_PATH=${KUBEVIRTCI_CONFIG_PATH:-$PWD} KUBECONFIG=${KUBECONFIG:-$BASE_PATH/$KUBEVIRT_PROVIDER/.kubeconfig} -GOCLI=${GOCLI:-${AAQ_DIR}/cluster-up/cli.sh} +GOCLI=${GOCLI:-${AAQ_DIR}/kubevirtci/cluster-up/cli.sh} KUBE_URL=${KUBE_URL:-""} AAQ_NAMESPACE=${AAQ_NAMESPACE:-aaq} -OPERATOR_CONTAINER_IMAGE=$(./cluster-up/kubectl.sh get deployment -n $AAQ_NAMESPACE aaq-operator -o'custom-columns=spec:spec.template.spec.containers[0].image' --no-headers) +OPERATOR_CONTAINER_IMAGE=$(./kubevirtci/cluster-up/kubectl.sh get deployment -n $AAQ_NAMESPACE aaq-operator -o'custom-columns=spec:spec.template.spec.containers[0].image' --no-headers) DOCKER_PREFIX=${OPERATOR_CONTAINER_IMAGE%/*} DOCKER_TAG=${OPERATOR_CONTAINER_IMAGE##*:} @@ -67,20 +67,20 @@ test_args="${test_args} -ginkgo.v ${arg_kubeurl} ${arg_namespace} ${arg_kubeco echo 'Wait until all AAQ Pods are ready' retry_counter=0 -while [ $retry_counter -lt $MAX_AAQ_WAIT_RETRY ] && [ -n "$(./cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE -o'custom-columns=status:status.containerStatuses[*].ready' --no-headers | grep false)" ]; do +while [ $retry_counter -lt $MAX_AAQ_WAIT_RETRY ] && [ -n "$(./kubevirtci/cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE -o'custom-columns=status:status.containerStatuses[*].ready' --no-headers | grep false)" ]; do retry_counter=$((retry_counter + 1)) sleep $AAQ_WAIT_TIME echo "Checking AAQ pods again, count $retry_counter" if [ $retry_counter -gt 1 ] && [ "$((retry_counter % 6))" -eq 0 ]; then - ./cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE + ./kubevirtci/cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE fi done if [ $retry_counter -eq $MAX_AAQ_WAIT_RETRY ]; then echo "Not all AAQ pods became ready" - ./cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE - ./cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE -o yaml - ./cluster-up/kubectl.sh describe pods -n $AAQ_NAMESPACE + ./kubevirtci/cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE + ./kubevirtci/cluster-up/kubectl.sh get pods -n $AAQ_NAMESPACE -o yaml + ./kubevirtci/cluster-up/kubectl.sh describe pods -n $AAQ_NAMESPACE exit 1 fi diff --git a/hack/bump-kubevirtci.sh b/hack/bump-kubevirtci.sh new file mode 100755 index 000000000..fbcbaedca --- /dev/null +++ b/hack/bump-kubevirtci.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +source $(dirname "$0")/config.sh + +val=$(curl -L https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirtci/latest) +sed -i "/^[[:blank:]]*kubevirtci_git_hash[[:blank:]]*=/s/=.*/=\"${val}\"/" hack/config.sh + +hack/sync-kubevirtci.sh \ No newline at end of file diff --git a/hack/cluster-up.sh b/hack/cluster-up.sh new file mode 100755 index 000000000..f969fe526 --- /dev/null +++ b/hack/cluster-up.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2021 Red Hat, Inc. +# + +set -e +set -x + +function install_kubevirt() { + if [ "$KUBEVIRT_RELEASE" = "latest_nightly" ]; then + LATEST=$(curl -L https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/latest) + kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-operator.yaml + kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-cr.yaml + elif [ "$KUBEVIRT_RELEASE" = "latest_stable" ]; then + RELEASE=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt) + kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-operator.yaml + kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-cr.yaml + else + kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-operator.yaml + kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-cr.yaml + fi + # Ensure the KubeVirt CRD is created + count=0 + until kubectl get crd kubevirts.kubevirt.io; do + ((count++)) && ((count == 30)) && echo "KubeVirt CRD not found" && exit 1 + echo "waiting for KubeVirt CRD" + sleep 1 + done + + # Ensure the KubeVirt API is available + count=0 + until kubectl api-resources --api-group=kubevirt.io | grep kubevirts; do + ((count++)) && ((count == 30)) && echo "KubeVirt API not found" && exit 1 + echo "waiting for KubeVirt API" + sleep 1 + done + + + # Ensure the KubeVirt CR is created + count=0 + until kubectl -n kubevirt get kv kubevirt; do + ((count++)) && ((count == 30)) && echo "KubeVirt CR not found" && exit 1 + echo "waiting for KubeVirt CR" + sleep 1 + done + + # Wait until KubeVirt is ready + count=0 + until kubectl wait -n kubevirt kv kubevirt --for condition=Available --timeout 5m; do + ((count++)) && ((count == 5)) && echo "KubeVirt not ready in time" && exit 1 + echo "Error waiting for KubeVirt to be Available, sleeping 1m and retrying" + sleep 1m + done +} + +source hack/config-kubevirtci.sh +KUBEVIRT_DEPLOY_CDI=${KUBEVIRT_DEPLOY_CDI:-true} +source "${KUBEVIRTCI_PATH}up.sh" +install_kubevirt \ No newline at end of file diff --git a/hack/config-kubevirtci.sh b/hack/config-kubevirtci.sh new file mode 100644 index 000000000..ac5bb1898 --- /dev/null +++ b/hack/config-kubevirtci.sh @@ -0,0 +1,23 @@ +# +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright the KubeVirt Authors. +# +# + +BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KUBEVIRTCI_PATH="${BASE_DIR}/kubevirtci/cluster-up/" +KUBEVIRTCI_CONFIG_PATH="${BASE_DIR}/kubevirtci/_ci-configs" +KUBEVIRT_RELEASE=${KUBEVIRT_RELEASE:-"latest_nightly"} \ No newline at end of file diff --git a/hack/config.sh b/hack/config.sh new file mode 100644 index 000000000..b9c931c77 --- /dev/null +++ b/hack/config.sh @@ -0,0 +1 @@ +kubevirtci_git_hash="2502262043-4fb13475" \ No newline at end of file diff --git a/hack/sync-kubevirtci.sh b/hack/sync-kubevirtci.sh new file mode 100755 index 000000000..6b6ee70a6 --- /dev/null +++ b/hack/sync-kubevirtci.sh @@ -0,0 +1,58 @@ + +#!/bin/bash + +set -ex + +# Required for kubevirtci_git_hash +source $(dirname "$0")/config.sh + +AAQ_DIR="$( + cd "$(dirname "$BASH_SOURCE[0]")/../" + pwd +)" + +# update cluster-up if needed +version_file="kubevirtci/cluster-up/version.txt" +sha_file="kubevirtci/cluster-up-sha.txt" +download_cluster_up=true +function getClusterUpShasum() { + ( + cd ${AAQ_DIR} + # We use LC_ALL=C to make sort canonical between machines, this is + # from sort man page [1]: + # ``` + # *** WARNING *** The locale specified by the environment affects sort + # order. Set LC_ALL=C to get the traditional sort order that uses + # native byte values. + # ``` + # [1] https://man7.org/linux/man-pages/man1/sort.1.html + find kubevirtci/cluster-up -type f | LC_ALL=C sort | xargs sha1sum | sha1sum | awk '{print $1}' + ) +} + +# check if we got a new cluster-up git commit hash +if [[ -f "${version_file}" ]] && [[ $(cat ${version_file}) == ${kubevirtci_git_hash} ]]; then + # check if files are modified + current_sha=$(getClusterUpShasum) + if [[ -f "${sha_file}" ]] && [[ $(cat ${sha_file}) == ${current_sha} ]]; then + echo "cluster-up is up to date and not modified" + download_cluster_up=false + else + echo "cluster-up was modified" + fi +else + echo "cluster-up git commit hash was updated" +fi +if [[ "$download_cluster_up" == true ]]; then + echo "downloading cluster-up" + rm -rf kubevirtci/cluster-up + ( + cd kubevirtci + curl --fail -L https://github.com/kubevirt/kubevirtci/archive/refs/tags/${kubevirtci_git_hash}.tar.gz | tar xz kubevirtci-${kubevirtci_git_hash}/cluster-up --strip-component 1 + ) + + echo ${kubevirtci_git_hash} >${version_file} + new_sha=$(getClusterUpShasum) + echo ${new_sha} >${sha_file} + echo "KUBEVIRTCI_TAG=${kubevirtci_git_hash}" >>kubevirtci/cluster-up/hack/common.sh +fi \ No newline at end of file diff --git a/hack/update-kubevirtci.sh b/hack/update-kubevirtci.sh deleted file mode 100755 index 20829b95c..000000000 --- a/hack/update-kubevirtci.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -# -# Copyright 2023 The AAQ Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_ROOT="$(cd "$(dirname $0)/../" && pwd -P)" - -# the kubevirtci tag to vendor from (https://github.com/kubevirt/kubevirtci/tags) -kubevirtci_release_tag=2403211043-970d172 - -# remove previous cluster-up dir entirely before vendoring -rm -rf ${SCRIPT_ROOT}/cluster-up - -# download and extract the cluster-up dir from a specific hash in kubevirtci -curl -L https://github.com/kubevirt/kubevirtci/archive/${kubevirtci_release_tag}/kubevirtci.tar.gz | tar xz kubevirtci-${kubevirtci_release_tag}/cluster-up --strip-component 1 - -echo "KUBEVIRTCI_TAG=${kubevirtci_release_tag}" >>${SCRIPT_ROOT}/cluster-up/hack/common.sh - -cat << 'EOF' >> ${SCRIPT_ROOT}/cluster-up/up.sh - -if [ "$KUBEVIRT_RELEASE" = "latest_nightly" ]; then - LATEST=$(curl -L https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/latest) - kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-operator.yaml - kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-cr.yaml -elif [ "$KUBEVIRT_RELEASE" = "latest_stable" ]; then - RELEASE=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt) - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-operator.yaml - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-cr.yaml -else - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-operator.yaml - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-cr.yaml -fi -# Ensure the KubeVirt CRD is created -count=0 -until kubectl get crd kubevirts.kubevirt.io; do - ((count++)) && ((count == 30)) && echo "KubeVirt CRD not found" && exit 1 - echo "waiting for KubeVirt CRD" - sleep 1 -done - -# Ensure the KubeVirt API is available -count=0 -until kubectl api-resources --api-group=kubevirt.io | grep kubevirts; do - ((count++)) && ((count == 30)) && echo "KubeVirt API not found" && exit 1 - echo "waiting for KubeVirt API" - sleep 1 -done - - -# Ensure the KubeVirt CR is created -count=0 -until kubectl -n kubevirt get kv kubevirt; do - ((count++)) && ((count == 30)) && echo "KubeVirt CR not found" && exit 1 - echo "waiting for KubeVirt CR" - sleep 1 -done - -# Wait until KubeVirt is ready -count=0 -until kubectl wait -n kubevirt kv kubevirt --for condition=Available --timeout 5m; do - ((count++)) && ((count == 5)) && echo "KubeVirt not ready in time" && exit 1 - echo "Error waiting for KubeVirt to be Available, sleeping 1m and retrying" - sleep 1m -done -EOF - diff --git a/kubevirtci/cluster-up-sha.txt b/kubevirtci/cluster-up-sha.txt new file mode 100644 index 000000000..68c461496 --- /dev/null +++ b/kubevirtci/cluster-up-sha.txt @@ -0,0 +1 @@ +90ca9c7e3735997caa34f593e82dd37547abafe0 diff --git a/cluster-up/.gitignore b/kubevirtci/cluster-up/.gitignore similarity index 100% rename from cluster-up/.gitignore rename to kubevirtci/cluster-up/.gitignore diff --git a/cluster-up/README.md b/kubevirtci/cluster-up/README.md similarity index 100% rename from cluster-up/README.md rename to kubevirtci/cluster-up/README.md diff --git a/cluster-up/check.sh b/kubevirtci/cluster-up/check.sh similarity index 75% rename from cluster-up/check.sh rename to kubevirtci/cluster-up/check.sh index dc64452aa..b285e4925 100755 --- a/cluster-up/check.sh +++ b/kubevirtci/cluster-up/check.sh @@ -26,19 +26,24 @@ fi KVM_ARCH="" KVM_NESTED="unknown" +KVM_HPAGE="unknown" if [ -f "/sys/module/kvm_intel/parameters/nested" ]; then KVM_NESTED=$( cat /sys/module/kvm_intel/parameters/nested ) KVM_ARCH="intel" elif [ -f "/sys/module/kvm_amd/parameters/nested" ]; then KVM_NESTED=$( cat /sys/module/kvm_amd/parameters/nested ) KVM_ARCH="amd" +elif [ -f "/sys/module/kvm/parameters/nested" ]; then + KVM_NESTED=$( cat /sys/module/kvm/parameters/nested ) + KVM_ARCH="s390x" + KVM_HPAGE=$( cat /sys/module/kvm/parameters/hpage ) fi function is_enabled() { if [ "$1" == "1" ]; then return 0 fi - if [ "$1" == "Y" ] || [ "$1" == "y"]; then + if [ "$1" == "Y" ] || [ "$1" == "y" ]; then return 0 fi return 1 @@ -49,3 +54,7 @@ if is_enabled "$KVM_NESTED"; then else echo "[ERR ] $KVM_ARCH nested virtualization not enabled" fi + +if is_enabled "$KVM_HPAGE" && [ "$(uname -m)" = "s390x" ]; then + echo "[ERR ] $KVM_HPAGE KVM hugepage enabled. It needs to be disabled while nested virtualization is enabled for s390x" +fi \ No newline at end of file diff --git a/cluster-up/cli.sh b/kubevirtci/cluster-up/cli.sh similarity index 100% rename from cluster-up/cli.sh rename to kubevirtci/cluster-up/cli.sh diff --git a/cluster-up/cluster/K8S.md b/kubevirtci/cluster-up/cluster/K8S.md similarity index 100% rename from cluster-up/cluster/K8S.md rename to kubevirtci/cluster-up/cluster/K8S.md diff --git a/cluster-up/cluster/K8S_AUTOMATION.md b/kubevirtci/cluster-up/cluster/K8S_AUTOMATION.md similarity index 100% rename from cluster-up/cluster/K8S_AUTOMATION.md rename to kubevirtci/cluster-up/cluster/K8S_AUTOMATION.md diff --git a/cluster-up/cluster/K8S_DEV_GUIDE.md b/kubevirtci/cluster-up/cluster/K8S_DEV_GUIDE.md similarity index 100% rename from cluster-up/cluster/K8S_DEV_GUIDE.md rename to kubevirtci/cluster-up/cluster/K8S_DEV_GUIDE.md diff --git a/cluster-up/cluster/README_VGPU.md b/kubevirtci/cluster-up/cluster/README_VGPU.md similarity index 100% rename from cluster-up/cluster/README_VGPU.md rename to kubevirtci/cluster-up/cluster/README_VGPU.md diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/kubevirtci/cluster-up/cluster/ephemeral-provider-common.sh similarity index 74% rename from cluster-up/cluster/ephemeral-provider-common.sh rename to kubevirtci/cluster-up/cluster/ephemeral-provider-common.sh index 577131c47..22a3fe75d 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/kubevirtci/cluster-up/cluster/ephemeral-provider-common.sh @@ -87,7 +87,7 @@ function _registry_volume() { function _add_common_params() { # shellcheck disable=SC2155 - local params="--nodes ${KUBEVIRT_NUM_NODES} --memory ${KUBEVIRT_MEMORY_SIZE} --cpu 6 --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --random-ports --background --prefix $provider_prefix ${KUBEVIRT_PROVIDER} ${KUBEVIRT_PROVIDER_EXTRA_ARGS}" + local params="--nodes ${KUBEVIRT_NUM_NODES} --memory ${KUBEVIRT_MEMORY_SIZE} --numa ${KUBEVIRT_NUM_NUMA_NODES} --cpu ${KUBEVIRT_NUM_VCPU} --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --random-ports --background --prefix $provider_prefix ${KUBEVIRT_PROVIDER} ${KUBEVIRT_PROVIDER_EXTRA_ARGS}" params=" --dns-port $KUBEVIRT_DNS_HOST_PORT $params" @@ -114,7 +114,8 @@ function _add_common_params() { params=" --container-suffix=:$KUBEVIRTCI_CONTAINER_SUFFIX $params" fi - if [[ ${KUBEVIRT_SLIM} == "true" ]]; then + # Currently, the s390x architecture supports only KUBEVIRT_SLIM. + if [[ ${KUBEVIRT_SLIM} == "true" || $(uname -m) == "s390x" ]]; then params=" --slim $params" fi fi @@ -138,10 +139,18 @@ function _add_common_params() { params=" --single-stack $params" fi + if [ $KUBEVIRT_NO_ETCD_FSYNC == "true" ]; then + params=" --no-etcd-fsync $params" + fi + if [ $KUBEVIRT_ENABLE_AUDIT == "true" ]; then params=" --enable-audit $params" fi + if [ $KUBVIRT_WITH_CNAO_SKIP_CONFIG == "true" ]; then + params=" --skip-cnao-cr $params" + fi + if [ $KUBEVIRT_DEPLOY_NFS_CSI == "true" ]; then params=" --enable-nfs-csi $params" fi @@ -169,6 +178,10 @@ function _add_common_params() { params=" --hugepages-2m $KUBEVIRT_HUGEPAGES_2M $params" fi + if [ -n "$KUBEVIRT_HUGEPAGES_1G" ]; then + params=" --hugepages-1g $KUBEVIRT_HUGEPAGES_1G $params" + fi + if [ -n "$KUBEVIRT_REALTIME_SCHEDULER" ]; then params=" --enable-realtime-scheduler $params" fi @@ -177,6 +190,59 @@ function _add_common_params() { params=" --enable-fips $params" fi + if [ "$KUBEVIRT_WITH_MULTUS_V3" == "true" ]; then + params=" --deploy-multus $params" + fi + + if [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then + params=" --enable-cnao $params" + fi + + if [ "$KUBEVIRT_DEPLOY_CDI" == "true" ]; then + params=" --deploy-cdi $params" + fi + + if [ -n "$KUBEVIRT_CUSTOM_CDI_VERSION" ]; then + params=" --cdi-version=$KUBEVIRT_CUSTOM_CDI_VERSION $params" + fi + + if [ "$KUBEVIRT_DEPLOY_AAQ" == "true" ]; then + params=" --deploy-aaq $params" + fi + + if [ -n "$KUBEVIRT_CUSTOM_AAQ_VERSION" ]; then + params=" --aaq-version=$KUBEVIRT_CUSTOM_AAQ_VERSION $params" + fi + + if [ "$KUBEVIRT_KSM_ON" == "true" ]; then + params=" --enable-ksm $params" + fi + + if [ ! -z $KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS ]; then + params=" --ksm-scan-interval=$KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS $params" + fi + + if [ ! -z $KUBEVIRT_KSM_PAGES_TO_SCAN ]; then + params=" --ksm-page-count=$KUBEVIRT_KSM_PAGES_TO_SCAN $params" + fi + + if [ "$KUBEVIRT_SWAP_ON" == "true" ]; then + params=" --enable-swap $params" + fi + + if [ ! -z $KUBEVIRT_SWAP_SIZE_IN_GB ]; then + params=" --swap-size=$KUBEVIRT_SWAP_SIZE_IN_GB $params" + fi + + if [ ! -z $KUBEVIRT_SWAPPINESS ]; then + params=" --swapiness=$KUBEVIRT_SWAPPINESS $params" + fi + + if [ $KUBEVIRT_UNLIMITEDSWAP == "true" ]; then + params=" --unlimited-swap $params" + fi + + if [ -n "$KUBEVIRTCI_PROXY" ]; then params=" --docker-proxy=$KUBEVIRTCI_PROXY $params" fi diff --git a/cluster-up/cluster/external/README.md b/kubevirtci/cluster-up/cluster/external/README.md similarity index 100% rename from cluster-up/cluster/external/README.md rename to kubevirtci/cluster-up/cluster/external/README.md diff --git a/cluster-up/cluster/external/provider.sh b/kubevirtci/cluster-up/cluster/external/provider.sh similarity index 82% rename from cluster-up/cluster/external/provider.sh rename to kubevirtci/cluster-up/cluster/external/provider.sh index ca9631025..59ae9852d 100644 --- a/cluster-up/cluster/external/provider.sh +++ b/kubevirtci/cluster-up/cluster/external/provider.sh @@ -22,6 +22,15 @@ manifest_docker_prefix=\${DOCKER_PREFIX} image_pull_policy=\${IMAGE_PULL_POLICY:-Always} EOF + if [ -n "$KUBECTL" ]; then + echo "kubectl=${KUBECTL}" >> "$PROVIDER_CONFIG_FILE_PATH" + else + if which kubectl; then + echo "kubectl=$(which kubectl)" >> "$PROVIDER_CONFIG_FILE_PATH" + fi + fi + + if which oc; then echo "oc=$(which oc)" >> "$PROVIDER_CONFIG_FILE_PATH" fi diff --git a/cluster-up/cluster/k8s-1.27/provider.sh b/kubevirtci/cluster-up/cluster/k8s-1.30/provider.sh similarity index 100% rename from cluster-up/cluster/k8s-1.27/provider.sh rename to kubevirtci/cluster-up/cluster/k8s-1.30/provider.sh diff --git a/cluster-up/cluster/k8s-1.28/provider.sh b/kubevirtci/cluster-up/cluster/k8s-1.31/provider.sh similarity index 100% rename from cluster-up/cluster/k8s-1.28/provider.sh rename to kubevirtci/cluster-up/cluster/k8s-1.31/provider.sh diff --git a/cluster-up/cluster/k8s-1.29/provider.sh b/kubevirtci/cluster-up/cluster/k8s-1.32/provider.sh similarity index 100% rename from cluster-up/cluster/k8s-1.29/provider.sh rename to kubevirtci/cluster-up/cluster/k8s-1.32/provider.sh diff --git a/kubevirtci/cluster-up/cluster/k8s-provider-common.sh b/kubevirtci/cluster-up/cluster/k8s-provider-common.sh new file mode 100644 index 000000000..ea752575a --- /dev/null +++ b/kubevirtci/cluster-up/cluster/k8s-provider-common.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -e + +# shellcheck source=cluster-up/cluster/ephemeral-provider-common.sh +source "${KUBEVIRTCI_PATH}/cluster/ephemeral-provider-common.sh" + + + +function deploy_kwok() { + if [[ ${KUBEVIRT_DEPLOY_KWOK} == "true" ]]; then + $kubectl create -f /opt/kwok/kwok.yaml + $kubectl create -f /opt/kwok/stage-fast.yaml + fi +} + +# copy_istio_cni_conf_files copy the generated Istio CNI net conf file +# (at '/etc/cni/multus/net.d/') to where Multus expect CNI net conf files ('/etc/cni/net.d/') +function copy_istio_cni_conf_files() { + if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ] && [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then + for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do + $ssh node${nodeNum} -- "until ls /etc/cni/multus > /dev/null 2>&1; do sleep 1; done" + $ssh node${nodeNum} -- sudo cp -uv /etc/cni/multus/net.d/*istio*.conf /etc/cni/net.d/ + done + fi +} + +# configure Prometheus to select kubevirt prometheusrules +function configure_prometheus() { + if [[ $KUBEVIRT_DEPLOY_PROMETHEUS == "true" ]] && $kubectl get crd prometheuses.monitoring.coreos.com; then + _kubectl patch prometheus k8s -n monitoring --type='json' -p='[{"op": "replace", "path": "/spec/ruleSelector", "value":{}}, {"op": "replace", "path": "/spec/ruleNamespaceSelector", "value":{"matchLabels": {}}}]' + fi +} + + +function wait_for_kwok_ready() { + if [ "KUBEVIRT_DEPLOY_KWOK" == "true" ]; then + $kubectl wait deployment -n kube-system kwok-controller --for condition=Available --timeout=200s + fi +} + +function up() { + params=$(_add_common_params) + if echo "$params" | grep -q ERROR; then + echo -e "$params" + exit 1 + fi + eval ${_cli:?} run $params + + ${_cli} scp --prefix $provider_prefix /etc/kubernetes/admin.conf - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig + + # Set server and disable tls check + export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig + kubectl config set-cluster kubernetes --server="https://$(_main_ip):$(_port k8s)" + kubectl config set-cluster kubernetes --insecure-skip-tls-verify=true + + ${_cli} scp --prefix ${provider_prefix:?} /usr/bin/kubectl - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl + + chmod u+x ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl + + # Make sure that local config is correct + prepare_config + ssh="${_cli} --prefix $provider_prefix ssh" + kubectl="$ssh node01 -- sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf" + + # For multinode cluster Label all the non control-plane nodes as workers, + # for one node cluster label control-plane with 'control-plane,worker' roles + if [ "$KUBEVIRT_NUM_NODES" -gt 1 ]; then + label="!node-role.kubernetes.io/control-plane" + else + label="node-role.kubernetes.io/control-plane" + fi + $kubectl label node -l $label node-role.kubernetes.io/worker='' + + configure_prometheus + + deploy_kwok + + until wait_for_kwok_ready; do + echo "Waiting for cluster components..." + sleep 5 + done + + # FIXME: remove 'copy_istio_cni_conf_files()' as soon as [1] and [2] are resolved + # [1] https://github.com/kubevirt/kubevirtci/issues/906 + # [2] https://github.com/k8snetworkplumbingwg/multus-cni/issues/982 + copy_istio_cni_conf_files +} diff --git a/cluster-up/cluster/kind-1.28/README.md b/kubevirtci/cluster-up/cluster/kind-1.28/README.md similarity index 100% rename from cluster-up/cluster/kind-1.28/README.md rename to kubevirtci/cluster-up/cluster/kind-1.28/README.md diff --git a/cluster-up/cluster/kind-1.27-vgpu/conformance.json b/kubevirtci/cluster-up/cluster/kind-1.28/conformance.json similarity index 100% rename from cluster-up/cluster/kind-1.27-vgpu/conformance.json rename to kubevirtci/cluster-up/cluster/kind-1.28/conformance.json diff --git a/cluster-up/cluster/kind-1.28/image b/kubevirtci/cluster-up/cluster/kind-1.28/image similarity index 100% rename from cluster-up/cluster/kind-1.28/image rename to kubevirtci/cluster-up/cluster/kind-1.28/image diff --git a/cluster-up/cluster/kind-1.28/provider.sh b/kubevirtci/cluster-up/cluster/kind-1.28/provider.sh similarity index 89% rename from cluster-up/cluster/kind-1.28/provider.sh rename to kubevirtci/cluster-up/cluster/kind-1.28/provider.sh index 0071f9fba..b46242133 100755 --- a/cluster-up/cluster/kind-1.28/provider.sh +++ b/kubevirtci/cluster-up/cluster/kind-1.28/provider.sh @@ -14,10 +14,10 @@ else fi function set_kind_params() { - version=$(cat cluster-up/cluster/$KUBEVIRT_PROVIDER/version) + version=$(cat "${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/version") export KIND_VERSION="${KIND_VERSION:-$version}" - image=$(cat cluster-up/cluster/$KUBEVIRT_PROVIDER/image) + image=$(cat "${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/image") export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-$image}" } @@ -37,6 +37,7 @@ function up() { cp $KIND_MANIFESTS_DIR/kind.yaml ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml _add_kubeadm_cpu_manager_config_patch _add_extra_mounts + _add_extra_portmapping export CONFIG_WORKER_CPU_MANAGER=true kind_up diff --git a/cluster-up/cluster/kind-1.28/version b/kubevirtci/cluster-up/cluster/kind-1.28/version similarity index 100% rename from cluster-up/cluster/kind-1.28/version rename to kubevirtci/cluster-up/cluster/kind-1.28/version diff --git a/cluster-up/cluster/kind-1.27-vgpu/config_vgpu_cluster.sh b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/config_vgpu_cluster.sh similarity index 100% rename from cluster-up/cluster/kind-1.27-vgpu/config_vgpu_cluster.sh rename to kubevirtci/cluster-up/cluster/kind-1.30-vgpu/config_vgpu_cluster.sh diff --git a/cluster-up/cluster/kind-sriov/conformance.json b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/conformance.json similarity index 100% rename from cluster-up/cluster/kind-sriov/conformance.json rename to kubevirtci/cluster-up/cluster/kind-1.30-vgpu/conformance.json diff --git a/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/image b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/image new file mode 100644 index 000000000..babda2aca --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/image @@ -0,0 +1 @@ +kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e diff --git a/cluster-up/cluster/kind-1.27-vgpu/provider.sh b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/provider.sh similarity index 89% rename from cluster-up/cluster/kind-1.27-vgpu/provider.sh rename to kubevirtci/cluster-up/cluster/kind-1.30-vgpu/provider.sh index 9d41c4717..5767a9195 100755 --- a/cluster-up/cluster/kind-1.27-vgpu/provider.sh +++ b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/provider.sh @@ -14,10 +14,10 @@ else fi function set_kind_params() { - version=$(cat cluster-up/cluster/$KUBEVIRT_PROVIDER/version) + version=$(cat "${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/version") export KIND_VERSION="${KIND_VERSION:-$version}" - image=$(cat cluster-up/cluster/$KUBEVIRT_PROVIDER/image) + image=$(cat "${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/image") export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-$image}" } @@ -34,9 +34,6 @@ function configure_registry_proxy() { } function up() { - # load the vfio_mdev module - /usr/sbin/modprobe vfio_mdev - # print hardware info for easier debugging based on logs echo 'Available cards' ${CRI_BIN} run --rm --cap-add=SYS_RAWIO quay.io/phoracek/lspci@sha256:0f3cacf7098202ef284308c64e3fc0ba441871a846022bb87d65ff130c79adb1 sh -c "lspci -k | grep -EA2 'VGA|3D'" diff --git a/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/version b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/version new file mode 100644 index 000000000..ca222b7cf --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/version @@ -0,0 +1 @@ +0.23.0 diff --git a/cluster-up/cluster/kind-1.27-vgpu/vgpu-node/node.sh b/kubevirtci/cluster-up/cluster/kind-1.30-vgpu/vgpu-node/node.sh similarity index 100% rename from cluster-up/cluster/kind-1.27-vgpu/vgpu-node/node.sh rename to kubevirtci/cluster-up/cluster/kind-1.30-vgpu/vgpu-node/node.sh diff --git a/kubevirtci/cluster-up/cluster/kind-1.31/README.md b/kubevirtci/cluster-up/cluster/kind-1.31/README.md new file mode 100644 index 000000000..bf48a0e4d --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-1.31/README.md @@ -0,0 +1,44 @@ +# K8S with mdev support in a Kind cluster + +Provides a pre-deployed k8s cluster that runs using [kind](https://github.com/kubernetes-sigs/kind) The cluster is completely ephemeral and is recreated on every cluster restart. +The KubeVirt containers are built on the local machine and are then pushed to a registry which is exposed at +`localhost:5000`. + +## Bringing the cluster up + +The following needs to be executed as root. Please refer to the name of the directory to get the kind version. + +```bash +export KUBEVIRT_PROVIDER=kind-x.yz +make cluster-up +``` + +The cluster can be accessed as usual: + +```bash +$ cluster-up/kubectl.sh get nodes +NAME STATUS ROLES AGE +kind-x.yz-control-plane Ready master 6m14s +``` + +## Bringing the cluster down + +```bash +make cluster-down +``` + +This destroys the whole cluster. + +## Setting a custom kind version + +In order to use a custom kind image / kind version, +export KIND_NODE_IMAGE, KIND_VERSION before running cluster-up. +For example in order to use kind 0.9.0 (which is based on k8s-1.19.1) use: +```bash +export KIND_NODE_IMAGE="kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600" +export KIND_VERSION="0.9.0" +``` +This allows users to test or use custom images / different kind versions before making them official. +See https://github.com/kubernetes-sigs/kind/releases for details about node images according to the kind version. + +- In order to use `make cluster-down` please make sure the right `CLUSTER_NAME` is exported. diff --git a/kubevirtci/cluster-up/cluster/kind-1.31/conformance.json b/kubevirtci/cluster-up/cluster/kind-1.31/conformance.json new file mode 100644 index 000000000..2ff6e83a5 --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-1.31/conformance.json @@ -0,0 +1,47 @@ +{ + "Description": "DEFAULT", + "UUID": "", + "Version": "v0.56.9", + "ResultsDir": "/tmp/sonobuoy/results", + "Resources": null, + "Filters": { + "Namespaces": ".*", + "LabelSelector": "" + }, + "Limits": { + "PodLogs": { + "Namespaces": "kube-system", + "SonobuoyNamespace": true, + "FieldSelectors": [], + "LabelSelector": "", + "Previous": false, + "SinceSeconds": null, + "SinceTime": null, + "Timestamps": false, + "TailLines": null, + "LimitBytes": null + } + }, + "QPS": 30, + "Burst": 50, + "Server": { + "bindaddress": "0.0.0.0", + "bindport": 8080, + "advertiseaddress": "", + "timeoutseconds": 21600 + }, + "Plugins": null, + "PluginSearchPath": [ + "./plugins.d", + "/etc/sonobuoy/plugins.d", + "~/sonobuoy/plugins.d" + ], + "Namespace": "sonobuoy", + "WorkerImage": "sonobuoy/sonobuoy:v0.56.9", + "ImagePullPolicy": "IfNotPresent", + "ImagePullSecrets": "", + "AggregatorPermissions": "clusterAdmin", + "ServiceAccountName": "sonobuoy-serviceaccount", + "ProgressUpdatesPort": "8099", + "SecurityContextMode": "nonroot" +} diff --git a/kubevirtci/cluster-up/cluster/kind-1.31/image b/kubevirtci/cluster-up/cluster/kind-1.31/image new file mode 100644 index 000000000..92a8c4480 --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-1.31/image @@ -0,0 +1 @@ +kindest/node:v1.31.2@sha256:18fbefc20a7113353c7b75b5c869d7145a6abd6269154825872dc59c1329912e diff --git a/kubevirtci/cluster-up/cluster/kind-1.31/provider.sh b/kubevirtci/cluster-up/cluster/kind-1.31/provider.sh new file mode 100755 index 000000000..3c67e97e6 --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-1.31/provider.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -e + +DEFAULT_CLUSTER_NAME="kind-1.31" +DEFAULT_HOST_PORT=5000 +ALTERNATE_HOST_PORT=5001 +export CLUSTER_NAME=${CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME} + +if [ $CLUSTER_NAME == $DEFAULT_CLUSTER_NAME ]; then + export HOST_PORT=$DEFAULT_HOST_PORT +else + export HOST_PORT=$ALTERNATE_HOST_PORT +fi + +function set_kind_params() { + version=$(cat "${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/version") + export KIND_VERSION="${KIND_VERSION:-$version}" + + image=$(cat "${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/image") + export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-$image}" +} + +function configure_registry_proxy() { + [ "$CI" != "true" ] && return + + echo "Configuring cluster nodes to work with CI mirror-proxy..." + + local -r ci_proxy_hostname="docker-mirror-proxy.kubevirt-prow.svc" + local -r kind_binary_path="${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kind" + local -r configure_registry_proxy_script="${KUBEVIRTCI_PATH}/cluster/kind/configure-registry-proxy.sh" + + KIND_BIN="$kind_binary_path" PROXY_HOSTNAME="$ci_proxy_hostname" $configure_registry_proxy_script +} + +function up() { + cp $KIND_MANIFESTS_DIR/kind.yaml ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml + _add_kubeadm_cpu_manager_config_patch + _add_extra_mounts + _add_extra_portmapping + export CONFIG_WORKER_CPU_MANAGER=true + kind_up + + configure_registry_proxy + + # remove the rancher.io kind default storageClass + _kubectl delete sc standard + + echo "$KUBEVIRT_PROVIDER cluster '$CLUSTER_NAME' is ready" +} + +set_kind_params + +source ${KUBEVIRTCI_PATH}/cluster/kind/common.sh diff --git a/kubevirtci/cluster-up/cluster/kind-1.31/version b/kubevirtci/cluster-up/cluster/kind-1.31/version new file mode 100644 index 000000000..d21d277be --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-1.31/version @@ -0,0 +1 @@ +0.25.0 diff --git a/kubevirtci/cluster-up/cluster/kind-ovn/README.md b/kubevirtci/cluster-up/cluster/kind-ovn/README.md new file mode 100644 index 000000000..11630ef88 --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-ovn/README.md @@ -0,0 +1,29 @@ +# OVN K8S in a Kind cluster + +Provides a k8s cluster that runs using [KinD](https://github.com/kubernetes-sigs/kind) +The cluster is completely ephemeral and is recreated on every cluster restart. The KubeVirt containers are built on the +local machine and are then pushed to a registry which is exposed at +`localhost:5000`. + +## Bringing the cluster up + +```bash +export KUBEVIRT_PROVIDER=kind-ovn +make cluster-up +``` + +## Bringing the cluster down + +```bash +export KUBEVIRT_PROVIDER=kind-ovn +make cluster-down +``` + +## FAQ + +In case the cluster deployment fails, you need to make sure you have enough watches +add those to /etc/sysctl.conf, and apply it `sysctl -p /etc/sysctl.conf`. +``` +sysctl fs.inotify.max_user_watches=1048576 +sysctl fs.inotify.max_user_instances=512 +``` diff --git a/kubevirtci/cluster-up/cluster/kind-ovn/install-ovn.sh b/kubevirtci/cluster-up/cluster/kind-ovn/install-ovn.sh new file mode 100755 index 000000000..f4a448eda --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-ovn/install-ovn.sh @@ -0,0 +1,47 @@ +#!/bin/bash -e +# +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2024 Red Hat, Inc. +# + +export OVNK_COMMIT=c77ee8c38c6a6d9e55131a1272db5fad5b606e44 + +OVNK_REPO='https://github.com/ovn-org/ovn-kubernetes.git' +CLUSTER_PATH=${CLUSTER_PATH:-"${KUBEVIRTCI_CONFIG_PATH}/${KUBEVIRT_PROVIDER}/_ovnk"} + +function cluster::_get_repo() { + git --git-dir ${CLUSTER_PATH}/.git config --get remote.origin.url +} + +function cluster::_get_sha() { + git --git-dir ${CLUSTER_PATH}/.git rev-parse HEAD +} + +function cluster::install() { + if [ -d ${CLUSTER_PATH} ]; then + if [ $(cluster::_get_repo) != ${OVNK_REPO} -o $(cluster::_get_sha) != ${OVNK_COMMIT} ]; then + rm -rf ${CLUSTER_PATH} + fi + fi + + if [ ! -d ${CLUSTER_PATH} ]; then + git clone ${OVNK_REPO} ${CLUSTER_PATH} + ( + cd ${CLUSTER_PATH} + git checkout ${OVNK_COMMIT} + ) + fi +} diff --git a/kubevirtci/cluster-up/cluster/kind-ovn/provider.sh b/kubevirtci/cluster-up/cluster/kind-ovn/provider.sh new file mode 100755 index 000000000..8a3506099 --- /dev/null +++ b/kubevirtci/cluster-up/cluster/kind-ovn/provider.sh @@ -0,0 +1,88 @@ +#!/bin/bash -ex +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2024 Red Hat, Inc. +# + +KIND_VERSION=0.19.0 +export KIND_IMAGE=kindest/node +export K8S_VERSION=v1.28.0@sha256:dad5a6238c5e41d7cac405fae3b5eda2ad1de6f1190fa8bfc64ff5bb86173213 + +KIND_PATH=${KIND_PATH:-"${KUBEVIRTCI_CONFIG_PATH}/${KUBEVIRT_PROVIDER}/_kind"} +CLUSTER_PATH=${CLUSTER_PATH:-"${KUBEVIRTCI_CONFIG_PATH}/${KUBEVIRT_PROVIDER}/_ovnk"} +CLUSTER_NAME=${KUBEVIRT_PROVIDER} + +function calculate_mtu() { + overlay_overhead=58 + current_mtu=$(cat /sys/class/net/$(ip route | grep "default via" | head -1 | awk '{print $5}')/mtu) + expr $current_mtu - $overlay_overhead +} + +MTU=${MTU:-$(calculate_mtu)} + +PLATFORM=$(uname -m) +case ${PLATFORM} in +x86_64* | i?86_64* | amd64*) + ARCH="amd64" + ;; +aarch64* | arm64*) + ARCH="arm64" + ;; +*) + echo "invalid Arch, only support x86_64, aarch64" + exit 1 + ;; +esac + +function fetch_kind() { + mkdir -p $KIND_PATH + current_kind_version=$($KIND_PATH/kind --version |& awk '{print $3}') + if [[ $current_kind_version != $KIND_VERSION ]]; then + echo "Downloading kind v$KIND_VERSION" + curl -LSs https://github.com/kubernetes-sigs/kind/releases/download/v$KIND_VERSION/kind-linux-${ARCH} -o "$KIND_PATH/kind" + chmod +x "$KIND_PATH/kind" + fi + export PATH=$KIND_PATH:$PATH +} + +function prepare_config() { + echo "STEP: Prepare provider config" + cat >$KUBEVIRTCI_CONFIG_PATH/$KUBEVIRT_PROVIDER/config-provider-$KUBEVIRT_PROVIDER.sh < ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig if ${CRI_BIN} exec ${CLUSTER_NAME}-control-plane ls /usr/bin/kubectl > /dev/null; then @@ -256,6 +268,22 @@ EOF fi } +function _add_extra_portmapping() { + if [[ "$KIND_PORT_MAPPING" != "" ]]; then + container_port=$(echo "$KIND_PORT_MAPPING" | awk -F: '{print $1}') + host_port=$(echo "$KIND_PORT_MAPPING" | awk -F: '{print $2}') + if [[ -z "$container_port" || -z "$host_port" ]]; then + echo "Invalid KIND_PORT_MAPPING format. Expected 'container_port:host_port'." + exit 1 + fi + cat <> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml + extraPortMappings: + - containerPort: $container_port + hostPort: $host_port +EOF + fi +} + function _add_kubeadm_cpu_manager_config_patch() { cat << EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml kubeadmConfigPatches: @@ -263,7 +291,6 @@ function _add_kubeadm_cpu_manager_config_patch() { kind: JoinConfiguration nodeRegistration: kubeletExtraArgs: - "feature-gates": "CPUManager=true" "cpu-manager-policy": "static" "kube-reserved": "cpu=500m" "system-reserved": "cpu=500m" @@ -283,11 +310,17 @@ EOF done } -function _add_kubeadm_config_patches() { - if [ $KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY == "true" ]; then - cat <> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml +function _add_kubeadm_config_patches_header() { + cat <> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml kubeadmConfigPatches: - | +EOF +} + +function _add_kubeadm_config_patches() { + _add_kubeadm_config_patches_header + if [ $KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY == "true" ]; then + cat <> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml kind: ClusterConfiguration metadata: name: config @@ -295,8 +328,16 @@ kubeadmConfigPatches: local: dataDir: $ETCD_IN_MEMORY_DATA_DIR EOF - echo "KIND cluster etcd data will be mounted to RAM on kind nodes: $ETCD_IN_MEMORY_DATA_DIR" - fi + echo "KIND cluster etcd data will be mounted to RAM on kind nodes: $ETCD_IN_MEMORY_DATA_DIR" + fi + if [[ -n "$CONFIG_TOPOLOGY_MANAGER_POLICY" ]]; then + cat <> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml + --- + kind: KubeletConfiguration + topologyManagerPolicy: ${CONFIG_TOPOLOGY_MANAGER_POLICY} + --- +EOF + fi } function _setup_ipfamily() { @@ -311,7 +352,9 @@ EOF function _prepare_kind_config() { _add_workers - _add_kubeadm_config_patches + if [[ "$KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY" == "true" || -n "$CONFIG_TOPOLOGY_MANAGER_POLICY" ]]; then + _add_kubeadm_config_patches + fi _setup_ipfamily echo "Final KIND config:" cat ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml diff --git a/cluster-up/cluster/kind/configure-registry-proxy.sh b/kubevirtci/cluster-up/cluster/kind/configure-registry-proxy.sh similarity index 100% rename from cluster-up/cluster/kind/configure-registry-proxy.sh rename to kubevirtci/cluster-up/cluster/kind/configure-registry-proxy.sh diff --git a/cluster-up/cluster/kind/manifests/kind.yaml b/kubevirtci/cluster-up/cluster/kind/manifests/kind.yaml similarity index 100% rename from cluster-up/cluster/kind/manifests/kind.yaml rename to kubevirtci/cluster-up/cluster/kind/manifests/kind.yaml diff --git a/cluster-up/cluster/kind/manifests/local-volume.yaml b/kubevirtci/cluster-up/cluster/kind/manifests/local-volume.yaml similarity index 100% rename from cluster-up/cluster/kind/manifests/local-volume.yaml rename to kubevirtci/cluster-up/cluster/kind/manifests/local-volume.yaml diff --git a/cluster-up/down.sh b/kubevirtci/cluster-up/down.sh similarity index 100% rename from cluster-up/down.sh rename to kubevirtci/cluster-up/down.sh diff --git a/cluster-up/hack/common.key b/kubevirtci/cluster-up/hack/common.key similarity index 100% rename from cluster-up/hack/common.key rename to kubevirtci/cluster-up/hack/common.key diff --git a/cluster-up/hack/common.sh b/kubevirtci/cluster-up/hack/common.sh similarity index 80% rename from cluster-up/hack/common.sh rename to kubevirtci/cluster-up/hack/common.sh index f7b3be261..3728e06a7 100644 --- a/cluster-up/hack/common.sh +++ b/kubevirtci/cluster-up/hack/common.sh @@ -17,13 +17,16 @@ fi KUBEVIRTCI_CLUSTER_PATH=${KUBEVIRTCI_CLUSTER_PATH:-${KUBEVIRTCI_PATH}/cluster} -KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.27} +KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.31} KUBEVIRT_NUM_NODES=${KUBEVIRT_NUM_NODES:-1} +KUBEVIRT_NUM_NUMA_NODES=${KUBEVIRT_NUM_NUMA_NODES:-1} +KUBEVIRT_NUM_VCPU=${KUBEVIRT_NUM_VCPU:-6} KUBEVIRT_MEMORY_SIZE=${KUBEVIRT_MEMORY_SIZE:-5120M} KUBEVIRT_NUM_SECONDARY_NICS=${KUBEVIRT_NUM_SECONDARY_NICS:-0} KUBEVIRT_DEPLOY_ISTIO=${KUBEVIRT_DEPLOY_ISTIO:-false} -KUBEVIRT_PSA=${KUBEVIRT_PSA:-false} +KUBEVIRT_PSA=${KUBEVIRT_PSA:-true} KUBEVIRT_SINGLE_STACK=${KUBEVIRT_SINGLE_STACK:-false} +KUBEVIRT_NO_ETCD_FSYNC=${KUBEVIRT_NO_ETCD_FSYNC:-false} KUBEVIRT_ENABLE_AUDIT=${KUBEVIRT_ENABLE_AUDIT:-false} KUBEVIRT_DEPLOY_NFS_CSI=${KUBEVIRT_DEPLOY_NFS_CSI:-false} KUBEVIRT_DEPLOY_PROMETHEUS=${KUBEVIRT_DEPLOY_PROMETHEUS:-false} @@ -31,10 +34,13 @@ KUBEVIRT_DEPLOY_PROMETHEUS_ALERTMANAGER=${KUBEVIRT_DEPLOY_PROMETHEUS_ALERTMANAGE KUBEVIRT_DEPLOY_GRAFANA=${KUBEVIRT_DEPLOY_GRAFANA:-false} KUBEVIRT_CGROUPV2=${KUBEVIRT_CGROUPV2:-false} KUBEVIRT_DEPLOY_CDI=${KUBEVIRT_DEPLOY_CDI:-false} +KUBEVIRT_DEPLOY_AAQ=${KUBEVIRT_DEPLOY_AAQ:-false} +KUBEVIRT_CUSTOM_AAQ_VERSION=${KUBEVIRT_CUSTOM_AAQ_VERSION} KUBEVIRT_CUSTOM_CDI_VERSION=${KUBEVIRT_CUSTOM_CDI_VERSION} KUBEVIRT_SWAP_ON=${KUBEVIRT_SWAP_ON:-false} KUBEVIRT_KSM_ON=${KUBEVIRT_KSM_ON:-false} KUBEVIRT_UNLIMITEDSWAP=${KUBEVIRT_UNLIMITEDSWAP:-false} +KUBVIRT_WITH_CNAO_SKIP_CONFIG=${KUBVIRT_WITH_CNAO_SKIP_CONFIG:-false} # If on a developer setup, expose ocp on 8443, so that the openshift web console can be used (the port is important because of auth redirects) # http and https ports are accessed by testing framework and should not be randomized @@ -48,4 +54,4 @@ provider_prefix=${JOB_NAME:-${KUBEVIRT_PROVIDER}}${EXECUTOR_NUMBER} job_prefix=${JOB_NAME:-kubevirt}${EXECUTOR_NUMBER} mkdir -p $KUBEVIRTCI_CONFIG_PATH/$KUBEVIRT_PROVIDER -KUBEVIRTCI_TAG=2403211043-970d172 +KUBEVIRTCI_TAG=2502262043-4fb13475 diff --git a/cluster-up/hack/config-default.sh b/kubevirtci/cluster-up/hack/config-default.sh similarity index 100% rename from cluster-up/hack/config-default.sh rename to kubevirtci/cluster-up/hack/config-default.sh diff --git a/cluster-up/hack/config.sh b/kubevirtci/cluster-up/hack/config.sh similarity index 100% rename from cluster-up/hack/config.sh rename to kubevirtci/cluster-up/hack/config.sh diff --git a/cluster-up/kubeconfig.sh b/kubevirtci/cluster-up/kubeconfig.sh similarity index 100% rename from cluster-up/kubeconfig.sh rename to kubevirtci/cluster-up/kubeconfig.sh diff --git a/cluster-up/kubectl.sh b/kubevirtci/cluster-up/kubectl.sh similarity index 100% rename from cluster-up/kubectl.sh rename to kubevirtci/cluster-up/kubectl.sh diff --git a/cluster-up/ssh.sh b/kubevirtci/cluster-up/ssh.sh similarity index 100% rename from cluster-up/ssh.sh rename to kubevirtci/cluster-up/ssh.sh diff --git a/kubevirtci/cluster-up/up.sh b/kubevirtci/cluster-up/up.sh new file mode 100755 index 000000000..fa48f152d --- /dev/null +++ b/kubevirtci/cluster-up/up.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +function validate_single_stack_ipv6() { + local kube_ns="kube-system" + local pod_label="calico-kube-controllers" + + echo "validating provider is single stack IPv6" + until _kubectl wait --for=condition=Ready pod --timeout=10s -n $kube_ns -lk8s-app=${pod_label}; do sleep 1; done > /dev/null 2>&1 + + local pod=$(_kubectl get pods -n ${kube_ns} -lk8s-app=${pod_label} -o=custom-columns=NAME:.metadata.name --no-headers) + local primary_ip=$(_kubectl get pod -n ${kube_ns} ${pod} -ojsonpath="{ @.status.podIP }") + + if [[ ! ${primary_ip} =~ fd00 ]]; then + echo "error: single stack primary ip ($primary_ip) is not IPv6 as expected" + exit 1 + fi + + if _kubectl get pod -n ${kube_ns} ${pod} -ojsonpath="{ @.status.podIPs[1] }" > /dev/null 2>&1; then + echo "error: single stack cluster expected" + exit 1 + fi +} + +if [ -z "$KUBEVIRTCI_PATH" ]; then + KUBEVIRTCI_PATH="$( + cd "$(dirname "$BASH_SOURCE[0]")/" + echo "$(pwd)/" + )" +fi + + +source ${KUBEVIRTCI_PATH}hack/common.sh +source ${KUBEVIRTCI_CLUSTER_PATH}/$KUBEVIRT_PROVIDER/provider.sh +up + +if [ ${KUBEVIRT_SINGLE_STACK} == true ]; then + validate_single_stack_ipv6 +fi diff --git a/kubevirtci/cluster-up/version.txt b/kubevirtci/cluster-up/version.txt new file mode 100644 index 000000000..2d3e0cde1 --- /dev/null +++ b/kubevirtci/cluster-up/version.txt @@ -0,0 +1 @@ +2502262043-4fb13475 diff --git a/cluster-up/virtctl.sh b/kubevirtci/cluster-up/virtctl.sh similarity index 68% rename from cluster-up/virtctl.sh rename to kubevirtci/cluster-up/virtctl.sh index 3c741bf41..1a65ab2ee 100755 --- a/cluster-up/virtctl.sh +++ b/kubevirtci/cluster-up/virtctl.sh @@ -19,6 +19,9 @@ set -e +>&2 echo "WARNING: usage of '${BASH_SOURCE[0]}' is deprecated!" +>&2 echo " see: https://github.com/kubevirt/kubevirtci/issues/1277" + if [ -z "$KUBEVIRTCI_PATH" ]; then KUBEVIRTCI_PATH="$( cd "$(dirname "$BASH_SOURCE[0]")/" @@ -38,5 +41,12 @@ elif [ -n "$KUBECONFIG" ]; then CONFIG_ARGS="--kubeconfig=${KUBECONFIG}" fi -${KUBEVIRTCI_PATH}/../_out/cmd/virtctl/virtctl $CONFIG_ARGS "$@" +KUBEVIRT_OUT_PATH=${KUBEVIRTCI_PATH}/../_out +if [ ! -d ${KUBEVIRT_OUT_PATH} ]; then + # see https://github.com/kubevirt/kubevirt/pull/12872 + >&2 echo "WARNING: $KUBEVIRT_OUT_PATH not found, falling back to parent" + KUBEVIRT_OUT_PATH=${KUBEVIRTCI_PATH}/../../_out + >&2 echo " $KUBEVIRT_OUT_PATH" +fi +${KUBEVIRT_OUT_PATH}/cmd/virtctl/virtctl $CONFIG_ARGS "$@" diff --git a/pkg/util/util.go b/pkg/util/util.go index 4a3bb081e..0f1e7efda 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -115,6 +115,11 @@ func CreateContainer(name, image, verbosity, pullPolicy string) corev1.Container container.TerminationMessagePath = corev1.TerminationMessagePathDefault container.Args = []string{"-v=" + verbosity} container.SecurityContext = &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + "ALL", + }, + }, SeccompProfile: &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeRuntimeDefault, }, diff --git a/tests/libaaq/aaq.go b/tests/libaaq/aaq.go index f1aebcd16..d1b250b22 100644 --- a/tests/libaaq/aaq.go +++ b/tests/libaaq/aaq.go @@ -6,6 +6,7 @@ import ( corev1 "k8s.io/api/core/v1" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/utils/pointer" "kubevirt.io/application-aware-quota/pkg/util" "kubevirt.io/application-aware-quota/staging/src/kubevirt.io/application-aware-quota-api/pkg/apis/core/v1alpha1" "kubevirt.io/application-aware-quota/tests/flags" @@ -42,6 +43,18 @@ func AddPlugablePolicy(aaq *v1alpha1.AAQ, pp PlugablePolicyName, c configName) * }, Image: plugablePolicyFor(pp), Args: []string{"--config", string(c)}, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + "ALL", + }, + }, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + AllowPrivilegeEscalation: pointer.Bool(false), + RunAsNonRoot: pointer.Bool(true), + }, }) return aaq