diff --git a/README.md b/README.md index 150faac..5b5b0b5 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ If you choose to install tools charts: - Redis - Dragonfly - Valkey +- Coredns # How to run diff --git a/charts/kuadrant-instances/templates/coredns/00-namespace.yaml b/charts/kuadrant-instances/templates/coredns/00-namespace.yaml new file mode 100644 index 0000000..5e76eb4 --- /dev/null +++ b/charts/kuadrant-instances/templates/coredns/00-namespace.yaml @@ -0,0 +1,7 @@ +{{ if .Values.coredns.enable }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.coredns.namespace }} +spec: {} +{{ end }} diff --git a/charts/tools-instances/templates/coredns/00-rbac.yaml b/charts/kuadrant-instances/templates/coredns/00-rbac.yaml similarity index 86% rename from charts/tools-instances/templates/coredns/00-rbac.yaml rename to charts/kuadrant-instances/templates/coredns/00-rbac.yaml index 71628fe..a2f3021 100644 --- a/charts/tools-instances/templates/coredns/00-rbac.yaml +++ b/charts/kuadrant-instances/templates/coredns/00-rbac.yaml @@ -1,9 +1,9 @@ -{{ if .Values.tools.coredns.enable }} +{{ if .Values.coredns.enable }} kind: ServiceAccount apiVersion: v1 metadata: name: coredns-service - namespace: {{ .Values.tools.namespace }} + namespace: {{ .Values.coredns.namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -47,5 +47,5 @@ roleRef: subjects: - kind: ServiceAccount name: coredns-service - namespace: {{ .Values.tools.namespace }} + namespace: {{ .Values.coredns.namespace }} {{ end }} diff --git a/charts/tools-instances/templates/coredns/01-deployment.yaml b/charts/kuadrant-instances/templates/coredns/01-deployment.yaml similarity index 86% rename from charts/tools-instances/templates/coredns/01-deployment.yaml rename to charts/kuadrant-instances/templates/coredns/01-deployment.yaml index de9e3e4..c627329 100644 --- a/charts/tools-instances/templates/coredns/01-deployment.yaml +++ b/charts/kuadrant-instances/templates/coredns/01-deployment.yaml @@ -1,12 +1,17 @@ -{{ if .Values.tools.coredns.enable }} +{{ if .Values.coredns.enable }} apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: coredns - namespace: {{ .Values.tools.namespace }} + namespace: {{ .Values.coredns.namespace }} labels: app: coredns spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 selector: matchLabels: app: coredns @@ -19,7 +24,7 @@ spec: serviceAccountName: coredns-service containers: - name: "coredns" - image: {{ .Values.tools.coredns.image }} + image: {{ .Values.coredns.image }} imagePullPolicy: Always env: - name: WATCH_NAMESPACES @@ -42,7 +47,7 @@ spec: - containerPort: 5300 name: "tcp-53" protocol: TCP - {{- if .Values.tools.coredns.metrics }} + {{- if .Values.coredns.metrics }} - containerPort: 9153 name: "tcp-9153" protocol: TCP @@ -78,11 +83,8 @@ spec: volumes: - name: config-volume configMap: - name: coredns + name: corefile items: - key: Corefile path: Corefile - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists {{ end }} diff --git a/charts/tools-instances/templates/coredns/03-service.yaml b/charts/kuadrant-instances/templates/coredns/03-service.yaml similarity index 51% rename from charts/tools-instances/templates/coredns/03-service.yaml rename to charts/kuadrant-instances/templates/coredns/03-service.yaml index 192497e..890c50c 100644 --- a/charts/tools-instances/templates/coredns/03-service.yaml +++ b/charts/kuadrant-instances/templates/coredns/03-service.yaml @@ -1,31 +1,9 @@ -{{ if .Values.tools.coredns.enable }} +{{ if .Values.coredns.enable }} apiVersion: v1 kind: Service metadata: - name: coredns-tcp - namespace: {{ .Values.tools.namespace }} - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - labels: - app: coredns -spec: - selector: - app: coredns - ports: - - name: "tcp-53" - protocol: TCP - port: 53 - targetPort: "tcp-53" - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: coredns-udp - namespace: {{ .Values.tools.namespace }} - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + name: coredns + namespace: {{ .Values.coredns.namespace }} labels: app: coredns spec: @@ -36,15 +14,18 @@ spec: protocol: UDP port: 53 targetPort: "udp-53" + - name: "tcp-53" + protocol: TCP + port: 53 + targetPort: "tcp-53" type: LoadBalancer - externalTrafficPolicy: Local --- -{{ if .Values.tools.coredns.metrics }} +{{ if .Values.coredns.metrics }} apiVersion: v1 kind: Service metadata: name: coredns-metrics - namespace: {{ .Values.tools.namespace }} + namespace: {{ .Values.coredns.namespace }} labels: app: coredns component: metrics diff --git a/charts/tools-instances/templates/coredns/04-corefile.yaml b/charts/kuadrant-instances/templates/coredns/04-corefile.yaml similarity index 56% rename from charts/tools-instances/templates/coredns/04-corefile.yaml rename to charts/kuadrant-instances/templates/coredns/04-corefile.yaml index c66dbcd..c24b6f1 100644 --- a/charts/tools-instances/templates/coredns/04-corefile.yaml +++ b/charts/kuadrant-instances/templates/coredns/04-corefile.yaml @@ -1,30 +1,24 @@ -{{ if .Values.tools.coredns.enable }} +{{ if .Values.coredns.enable }} apiVersion: v1 kind: ConfigMap metadata: - name: coredns - namespace: {{ .Values.tools.namespace }} + name: corefile + namespace: {{ .Values.coredns.namespace }} labels: app: coredns data: Corefile: |- - kdrnt { - debug - errors + . { health { lameduck 5s } ready - log - transfer { - to * - } - kuadrant - prometheus 0.0.0.0:9153 } - {{ .Values.tools.coredns.zone }} { + {{- range .Values.coredns.zones }} + {{ . }} { debug errors + ready log geoip GeoLite2-City-demo.mmdb { edns-subnet @@ -33,7 +27,11 @@ data: transfer { to * } - kuadrant + kuadrant { + rname kuadrant-qe-infra@redhat.com + } prometheus 0.0.0.0:9153 } + {{ end }} + {{ end }} diff --git a/charts/tools-instances/templates/coredns/05-serviceMonitor.yaml b/charts/kuadrant-instances/templates/coredns/05-serviceMonitor.yaml similarity index 63% rename from charts/tools-instances/templates/coredns/05-serviceMonitor.yaml rename to charts/kuadrant-instances/templates/coredns/05-serviceMonitor.yaml index e1ec99a..e9bb267 100644 --- a/charts/tools-instances/templates/coredns/05-serviceMonitor.yaml +++ b/charts/kuadrant-instances/templates/coredns/05-serviceMonitor.yaml @@ -1,15 +1,15 @@ -{{ if and .Values.tools.coredns.enable .Values.tools.coredns.metrics }} +{{ if and .Values.coredns.enable .Values.coredns.metrics }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: coredns - namespace: {{ .Values.tools.namespace }} + namespace: {{ .Values.coredns.namespace }} labels: app: coredns spec: namespaceSelector: matchNames: - - {{ .Values.tools.namespace }} + - {{ .Values.coredns.namespace }} selector: matchLabels: app: coredns diff --git a/charts/tools-instances/templates/coredns/06-remoteServiceAccount.yaml b/charts/kuadrant-instances/templates/coredns/06-remoteServiceAccount.yaml similarity index 81% rename from charts/tools-instances/templates/coredns/06-remoteServiceAccount.yaml rename to charts/kuadrant-instances/templates/coredns/06-remoteServiceAccount.yaml index e0abfba..20fd39f 100644 --- a/charts/tools-instances/templates/coredns/06-remoteServiceAccount.yaml +++ b/charts/kuadrant-instances/templates/coredns/06-remoteServiceAccount.yaml @@ -1,9 +1,9 @@ -{{ if .Values.tools.coredns.enable }} +{{ if and .Values.coredns.enable .Values.tools.enabled }} kind: ServiceAccount apiVersion: v1 metadata: name: coredns - namespace: {{ .Values.tools.namespace }} + namespace: {{ .Values.coredns.namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -38,5 +38,5 @@ roleRef: subjects: - kind: ServiceAccount name: coredns - namespace: {{ .Values.tools.namespace }} + namespace: {{ .Values.coredns.namespace }} {{ end }} diff --git a/charts/kuadrant-instances/templates/coredns/autodelegate/00-rbac.yaml b/charts/kuadrant-instances/templates/coredns/autodelegate/00-rbac.yaml new file mode 100644 index 0000000..4ce667c --- /dev/null +++ b/charts/kuadrant-instances/templates/coredns/autodelegate/00-rbac.yaml @@ -0,0 +1,43 @@ +{{ if and .Values.coredns.enable .Values.coredns.autoDelegate }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: post-install-autodelegate-sa + namespace: {{ .Values.coredns.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: post-install-autodelegate-role + namespace: {{ .Values.coredns.namespace }} +rules: + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: post-install-autodelegate-rb + namespace: {{ .Values.coredns.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: post-install-autodelegate-role +subjects: + - kind: ServiceAccount + name: post-install-autodelegate-sa + namespace: {{ .Values.coredns.namespace }} +{{ end }} diff --git a/charts/kuadrant-instances/templates/coredns/autodelegate/01-configMap-script.yaml b/charts/kuadrant-instances/templates/coredns/autodelegate/01-configMap-script.yaml new file mode 100644 index 0000000..c918779 --- /dev/null +++ b/charts/kuadrant-instances/templates/coredns/autodelegate/01-configMap-script.yaml @@ -0,0 +1,128 @@ +{{ if and .Values.coredns.enable .Values.coredns.autoDelegate }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: post-install-autodelegate + namespace: {{ .Values.coredns.namespace }} +data: + "create.sh": | + #!/bin/bash + set -xe + export AWS_RETRY_MODE="adaptive" + export AWS_MAX_ATTEMPTS="10" + NS_NAME="{{ .Values.coredns.nameserverName }}" + TTL=200 + + kubectl wait -n {{ .Values.coredns.namespace }} service coredns --timeout=120s --for=jsonpath='{.status.loadBalancer.ingress}' || exit 1 + COREDNS_IP="$(kubectl get -n {{ .Values.coredns.namespace }} service coredns '-o=jsonpath={.status.loadBalancer.ingress[0].ip}')" + COREDNS_HOSTNAME="$(kubectl get -n {{ .Values.coredns.namespace }} service coredns '-o=jsonpath={.status.loadBalancer.ingress[0].hostname}')" + + {{- range .Values.coredns.zones }} + ZONE="{{ . }}" + + if [ -n "$COREDNS_IP" ]; then + # Create A record for the nameserver + CHANGE_BATCH=$(jq -n \ + --arg name "${NS_NAME}.${ZONE}" \ + --arg ip "$COREDNS_IP" \ + --argjson ttl "$TTL" \ + '{Changes: [{Action: "UPSERT", ResourceRecordSet: {Name: $name, Type: "A", TTL: $ttl, ResourceRecords: [{Value: $ip}]}}]}') + CHANGE_ID=$(aws route53 change-resource-record-sets --hosted-zone-id "${ZONE_ID}" --change-batch "$CHANGE_BATCH" \ + --query 'ChangeInfo.Id' --output text) + aws route53 wait resource-record-sets-changed --id "$CHANGE_ID" + + NS_VALUE="${NS_NAME}.${ZONE}" + else + NS_VALUE="${COREDNS_HOSTNAME}." + fi + + # Get current NS records for the zone (if any) + CURRENT_NS=$(aws route53 list-resource-record-sets --hosted-zone-id "${ZONE_ID}" \ + --query "ResourceRecordSets[?Name=='${ZONE}' && Type=='NS']" --output json) + + if [ "$CURRENT_NS" == "[]" ]; then + # No existing NS records, create new + NEW_RECORDS=$(jq -n --arg value "$NS_VALUE" '[{Value: $value}]') + else + # Append to existing NS records + CURRENT_TTL=$(echo "$CURRENT_NS" | jq -r '.[0].TTL') + TTL=$CURRENT_TTL + NEW_RECORDS=$(echo "$CURRENT_NS" | jq --arg value "$NS_VALUE" \ + '.[0].ResourceRecords + [{Value: $value}] | unique_by(.Value)') + fi + + # Upsert NS record with combined values + CHANGE_BATCH=$(jq -n \ + --arg name "${ZONE}" \ + --argjson ttl "$TTL" \ + --argjson records "$NEW_RECORDS" \ + '{Changes: [{Action: "UPSERT", ResourceRecordSet: {Name: $name, Type: "NS", TTL: $ttl, ResourceRecords: $records}}]}') + CHANGE_ID=$(aws route53 change-resource-record-sets --hosted-zone-id "${ZONE_ID}" --change-batch "$CHANGE_BATCH" \ + --query 'ChangeInfo.Id' --output text) + aws route53 wait resource-record-sets-changed --id "$CHANGE_ID" + {{ end }} + "delete.sh": | + #!/bin/bash + set -xe + export AWS_RETRY_MODE="adaptive" + export AWS_MAX_ATTEMPTS="10" + NS_NAME="{{ .Values.coredns.nameserverName }}" + + COREDNS_IP="$(kubectl get -n {{ .Values.coredns.namespace }} service coredns '-o=jsonpath={.status.loadBalancer.ingress[0].ip}')" + COREDNS_HOSTNAME="$(kubectl get -n {{ .Values.coredns.namespace }} service coredns '-o=jsonpath={.status.loadBalancer.ingress[0].hostname}')" + + {{- range .Values.coredns.zones }} + ZONE="{{ . }}" + + # Delete A record if using IP + if [ -n "$COREDNS_IP" ]; then + EXCLUDE_VALUE="${NS_NAME}.${ZONE}" + CURRENT_A=$(aws route53 list-resource-record-sets --hosted-zone-id "${ZONE_ID}" \ + --query "ResourceRecordSets[?Name=='${NS_NAME}.${ZONE}' && Type=='A']" --output json) + if [ "$CURRENT_A" != "[]" ]; then + TTL=$(echo "$CURRENT_A" | jq -r '.[0].TTL') + CHANGE_BATCH=$(jq -n \ + --arg name "${NS_NAME}.${ZONE}" \ + --arg ip "$COREDNS_IP" \ + --argjson ttl "$TTL" \ + '{Changes: [{Action: "DELETE", ResourceRecordSet: {Name: $name, Type: "A", TTL: $ttl, ResourceRecords: [{Value: $ip}]}}]}') + aws route53 change-resource-record-sets --hosted-zone-id "${ZONE_ID}" --change-batch "$CHANGE_BATCH" + fi + else + EXCLUDE_VALUE="${COREDNS_HOSTNAME}." + fi + + # Get current NS records for the zone + CURRENT_NS=$(aws route53 list-resource-record-sets --hosted-zone-id "${ZONE_ID}" \ + --query "ResourceRecordSets[?Name=='${ZONE}' && Type=='NS']" --output json) + + if [ "$CURRENT_NS" != "[]" ]; then + TTL=$(echo "$CURRENT_NS" | jq -r '.[0].TTL') + CURRENT_RECORDS=$(echo "$CURRENT_NS" | jq '.[0].ResourceRecords') + + # Extract current values, filter out the one to exclude + REMAINING_VALUES=$(echo "$CURRENT_NS" | jq --arg exclude "${EXCLUDE_VALUE}" \ + '.[0].ResourceRecords | map(select(.Value != $exclude))') + REMAINING_COUNT=$(echo "$REMAINING_VALUES" | jq 'length') + + # possible race condition here; if multiple deleting clusters enter this part simultaneously, there will be a leftover NS record in route53 + if [ "$REMAINING_COUNT" -eq 0 ]; then + # No remaining values, delete the NS record entirely + CHANGE_BATCH=$(jq -n \ + --arg name "${ZONE}" \ + --argjson ttl "$TTL" \ + --argjson records "$CURRENT_RECORDS" \ + '{Changes: [{Action: "DELETE", ResourceRecordSet: {Name: $name, Type: "NS", TTL: $ttl, ResourceRecords: $records}}]}') + aws route53 change-resource-record-sets --hosted-zone-id "${ZONE_ID}" --change-batch "$CHANGE_BATCH" + else + # Update with remaining values only + CHANGE_BATCH=$(jq -n \ + --arg name "${ZONE}" \ + --argjson ttl "$TTL" \ + --argjson records "$REMAINING_VALUES" \ + '{Changes: [{Action: "UPSERT", ResourceRecordSet: {Name: $name, Type: "NS", TTL: $ttl, ResourceRecords: $records}}]}') + aws route53 change-resource-record-sets --hosted-zone-id "${ZONE_ID}" --change-batch "$CHANGE_BATCH" + fi + fi + {{ end }} +{{ end }} diff --git a/charts/kuadrant-instances/templates/coredns/autodelegate/02-create-job.yaml b/charts/kuadrant-instances/templates/coredns/autodelegate/02-create-job.yaml new file mode 100644 index 0000000..1176c4a --- /dev/null +++ b/charts/kuadrant-instances/templates/coredns/autodelegate/02-create-job.yaml @@ -0,0 +1,32 @@ +{{ if and .Values.coredns.enable .Values.coredns.autoDelegate }} +apiVersion: batch/v1 +kind: Job +metadata: + name: post-install-autodelegate + namespace: {{ .Values.coredns.namespace }} + annotations: + "helm.sh/hook": post-install +spec: + backoffLimit: 3 + template: + spec: + containers: + - command: + - /bin/bash + - /scripts/create.sh + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + name: post-install + volumeMounts: + - name: script-volume + mountPath: /scripts + resources: {} + envFrom: + - secretRef: + name: coredns-autodelegate + volumes: + - name: script-volume + configMap: + name: post-install-autodelegate + serviceAccountName: post-install-autodelegate-sa + restartPolicy: OnFailure +{{ end }} diff --git a/charts/kuadrant-instances/templates/coredns/autodelegate/04-delete-job.yaml b/charts/kuadrant-instances/templates/coredns/autodelegate/04-delete-job.yaml new file mode 100644 index 0000000..e4ff359 --- /dev/null +++ b/charts/kuadrant-instances/templates/coredns/autodelegate/04-delete-job.yaml @@ -0,0 +1,32 @@ +{{ if and .Values.coredns.enable .Values.coredns.autoDelegate }} +apiVersion: batch/v1 +kind: Job +metadata: + name: autodelegate-cleanup + namespace: {{ .Values.coredns.namespace }} + annotations: + "helm.sh/hook": pre-delete +spec: + backoffLimit: 10 + template: + spec: + containers: + - command: + - /bin/bash + - /scripts/delete.sh + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + name: cleanup + volumeMounts: + - name: script-volume + mountPath: /scripts + resources: {} + envFrom: + - secretRef: + name: coredns-autodelegate + volumes: + - name: script-volume + configMap: + name: post-install-autodelegate + serviceAccountName: post-install-autodelegate-sa + restartPolicy: OnFailure +{{ end }} diff --git a/charts/kuadrant-instances/templates/coredns/autodelegate/05-coredns-config-secret.yaml b/charts/kuadrant-instances/templates/coredns/autodelegate/05-coredns-config-secret.yaml new file mode 100644 index 0000000..b893414 --- /dev/null +++ b/charts/kuadrant-instances/templates/coredns/autodelegate/05-coredns-config-secret.yaml @@ -0,0 +1,12 @@ +{{ if and .Values.coredns.enable .Values.coredns.autoDelegate .Values.tools.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: coredns-credentials + namespace: kuadrant + annotations: + base_domain: {{ first .Values.coredns.zones | trimSuffix "." }} +type: kuadrant.io/coredns +stringData: + ZONES: {{ .Values.coredns.zones | join "," | trimSuffix "." | quote }} +{{ end }} diff --git a/example-additionalManifests.yaml b/example-additionalManifests.yaml index bf271cb..ace32a7 100644 --- a/example-additionalManifests.yaml +++ b/example-additionalManifests.yaml @@ -129,3 +129,14 @@ preOperatorInstallManifests: data: .dockerconfigjson: type: kubernetes.io/dockerconfigjson +toolsManifests: + - apiVersion: v1 + kind: Secret + metadata: + name: coredns-autodelegate + namespace: tools + stringData: + AWS_ACCESS_KEY_ID: + ZONE_ID: + AWS_SECRET_ACCESS_KEY: + type: Opaque diff --git a/install.sh b/install.sh index ff47e42..830f44d 100755 --- a/install.sh +++ b/install.sh @@ -25,10 +25,10 @@ eval "$helm_cmd" if [[ "$1" == "-t" ]]; then echo "--Installing tools operators" -helm install --wait tools-operators charts/tools-operators +helm install --values additionalManifests.yaml --wait tools-operators charts/tools-operators echo "--Installing tools instances" -helm install --wait --timeout 10m tools-instances charts/tools-instances +helm install --values additionalManifests.yaml --wait --timeout 10m tools-instances charts/tools-instances fi echo "Success!" diff --git a/tools-install.sh b/tools-install.sh index 0a3354d..3784031 100755 --- a/tools-install.sh +++ b/tools-install.sh @@ -5,7 +5,7 @@ set -e; set -o pipefail; cd "$(dirname "$0")" -additional_flags='' +additional_flags='--values additionalManifests.yaml' if [[ "$1" == "-k" ]]; then additional_flags+=" --set tools.keycloak.keycloakProvider=deployment" diff --git a/values-tools.yaml b/values-tools.yaml index afec996..7dd56a6 100644 --- a/values-tools.yaml +++ b/values-tools.yaml @@ -31,8 +31,3 @@ tools: customMetricsApiserver: enable: true image: quay.io/acristur/custom-metrics-apiserver:latest - coredns: - enable: false - metrics: false - zone: "k.example.com" - image: quay.io/kuadrant/coredns-kuadrant:latest diff --git a/values.yaml b/values.yaml index 9a7e12b..8960bd8 100644 --- a/values.yaml +++ b/values.yaml @@ -84,6 +84,20 @@ certManager: startingCSV: cert-manager-operator.v1.15.0 # Ignored if 'Automatic' bellow installPlanApproval: Automatic # can be 'Automatic' or 'Manual' +# Coredns +coredns: + enable: false + namespace: kuadrant-coredns + metrics: false + # be sure to include the root domain dot "." + zones: + - "k.example.com." + # Autodelegation script will delegate to route53 if set to true + # You need to have `coredns-autodelegate` secret in additionalManifests + autoDelegate: false + nameserverName: "ns1" + image: quay.io/kuadrant/coredns-kuadrant:latest + # Optional objects used for Kuadrant testing # Note: For setting tools values go to values-tools.yaml tools: