diff --git a/hack/go-unit-tests-vpa.sh b/hack/go-unit-tests-vpa.sh index c4bec3971495..457914bb79be 100755 --- a/hack/go-unit-tests-vpa.sh +++ b/hack/go-unit-tests-vpa.sh @@ -21,7 +21,7 @@ set -o nounset CONTRIB_ROOT="$(dirname ${BASH_SOURCE})/.." pushd ${CONTRIB_ROOT}/vertical-pod-autoscaler -go test -count=1 -race $(go list ./... | grep -v /vendor/ | grep -v vertical-pod-autoscaler/e2e) +go test -count=1 -race $(go list ./... | grep -v /vendor/ | grep -v vertical-pod-autoscaler/e2e | grep -v vertical-pod-autoscaler/integration) popd pushd ${CONTRIB_ROOT}/vertical-pod-autoscaler/e2e go test -run=None ./... diff --git a/vertical-pod-autoscaler/common/flags.go b/vertical-pod-autoscaler/common/flags.go index fc449d2ae60e..ef6d8b5dfb54 100644 --- a/vertical-pod-autoscaler/common/flags.go +++ b/vertical-pod-autoscaler/common/flags.go @@ -35,18 +35,38 @@ type CommonFlags struct { IgnoredVpaObjectNamespaces string } +// DefaultCommonConfig returns the default values for common flags +func DefaultCommonConfig() *CommonFlags { + return &CommonFlags{ + KubeConfig: "", + KubeApiQps: 50.0, + KubeApiBurst: 100.0, + EnableProfiling: false, + VpaObjectNamespace: apiv1.NamespaceAll, + IgnoredVpaObjectNamespaces: "", + } +} + // InitCommonFlags initializes the common flags func InitCommonFlags() *CommonFlags { - cf := &CommonFlags{} - flag.StringVar(&cf.KubeConfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") - flag.Float64Var(&cf.KubeApiQps, "kube-api-qps", 50.0, "QPS limit when making requests to Kubernetes apiserver") - flag.Float64Var(&cf.KubeApiBurst, "kube-api-burst", 100.0, "QPS burst limit when making requests to Kubernetes apiserver") - flag.BoolVar(&cf.EnableProfiling, "profiling", false, "Is debug/pprof endpoint enabled") - flag.StringVar(&cf.VpaObjectNamespace, "vpa-object-namespace", apiv1.NamespaceAll, "Specifies the namespace to search for VPA objects. Leave empty to include all namespaces. If provided, the garbage collector will only clean this namespace.") - flag.StringVar(&cf.IgnoredVpaObjectNamespaces, "ignored-vpa-object-namespaces", "", "A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector.") + cf := DefaultCommonConfig() + flag.StringVar(&cf.KubeConfig, "kubeconfig", cf.KubeConfig, "Path to a kubeconfig. Only required if out-of-cluster.") + flag.Float64Var(&cf.KubeApiQps, "kube-api-qps", cf.KubeApiQps, "QPS limit when making requests to Kubernetes apiserver") + flag.Float64Var(&cf.KubeApiBurst, "kube-api-burst", cf.KubeApiBurst, "QPS burst limit when making requests to Kubernetes apiserver") + flag.BoolVar(&cf.EnableProfiling, "profiling", cf.EnableProfiling, "Is debug/pprof endpoint enabled") + flag.StringVar(&cf.VpaObjectNamespace, "vpa-object-namespace", cf.VpaObjectNamespace, "Specifies the namespace to search for VPA objects. Leave empty to include all namespaces. If provided, the garbage collector will only clean this namespace.") + flag.StringVar(&cf.IgnoredVpaObjectNamespaces, "ignored-vpa-object-namespaces", cf.IgnoredVpaObjectNamespaces, "A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector.") return cf } +// ValidateCommonConfig performs validation of the common flags +func ValidateCommonConfig(config *CommonFlags) { + if len(config.VpaObjectNamespace) > 0 && len(config.IgnoredVpaObjectNamespaces) > 0 { + klog.ErrorS(nil, "--vpa-object-namespace and --ignored-vpa-object-namespaces are mutually exclusive and can't be set together.") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } +} + // InitLoggingFlags initializes the logging flags func InitLoggingFlags() { // Set the default log level to 4 (info) diff --git a/vertical-pod-autoscaler/docs/development-and-testing.md b/vertical-pod-autoscaler/docs/development-and-testing.md index 9d4cec107f3b..db4e0796cf55 100644 --- a/vertical-pod-autoscaler/docs/development-and-testing.md +++ b/vertical-pod-autoscaler/docs/development-and-testing.md @@ -4,6 +4,7 @@ - [Introduction](#introduction) +- [Running integration tests](#running-integration-tests) - [Running e2e tests](#running-e2e-tests) - [Feature gates](#feature-gates) - [Parallelism](#parallelism) @@ -14,6 +15,34 @@ This project contains various scripts and tools to aid in the development of the three VPA components. +## Running integration tests + +The VPA contains integration tests that test individual components in isolation using [controller-runtime's envtest](https://github.com/kubernetes-sigs/controller-runtime/tree/main/tools/setup-envtest) environment. + +They can be run using the `./hack/run-integration-tests.sh` helper script. This script automatically downloads and manages the required Kubernetes API server and etcd binaries for testing. + +Integration tests are faster than e2e tests and do not require a full Kubernetes cluster. They are suitable for testing: + +- Component-specific behavior and logic +- Configuration handling +- API object validation +- Resource filtering and selection logic + +**Example usage:** + +```bash +# Run all integration tests +./hack/run-integration-tests.sh + +# Run a specific test +./hack/run-integration-tests.sh -run TestRecommenderWithNamespaceFiltering + +# Run tests with custom parallelism +./hack/run-integration-tests.sh -parallel 8 +``` + +By default, integration tests run with 4 parallel workers. + ## Running e2e tests The VPA contains some e2e tests that test how each component interacts with Pods and VPA resources inside a real Kubernetes cluster. diff --git a/vertical-pod-autoscaler/docs/flags.md b/vertical-pod-autoscaler/docs/flags.md index cd0f2b16441b..ba8a4f545f69 100644 --- a/vertical-pod-autoscaler/docs/flags.md +++ b/vertical-pod-autoscaler/docs/flags.md @@ -148,7 +148,7 @@ This document is auto-generated from the flag definitions in the VPA updater cod | `alsologtostderr` | | | log to standard error as well as files (no effect when -logtostderr=true) | | `evict-after-oom-threshold` | | 10m0s | duration Evict pod that has OOMed in less than evict-after-oom-threshold since start. | | `eviction-rate-burst` | int | 1 | Burst of pods that can be evicted. | -| `eviction-rate-limit` | float | | Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable
the rate limiter. (default -1) | +| `eviction-rate-limit` | float | -1 | Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable the rate limiter. | | `eviction-tolerance` | float | 0.5 | Fraction of replica count that can be evicted for update, if more than one pod can be evicted. | | `feature-gates` | mapStringBool | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
AllAlpha=true\|false (ALPHA - default=false)
AllBeta=true\|false (BETA - default=false)
PerVPAConfig=true\|false (ALPHA - default=false) | | `ignored-vpa-object-namespaces` | string | | A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. | diff --git a/vertical-pod-autoscaler/e2e/go.mod b/vertical-pod-autoscaler/e2e/go.mod index 09a0c52a4928..1cf580c08e60 100644 --- a/vertical-pod-autoscaler/e2e/go.mod +++ b/vertical-pod-autoscaler/e2e/go.mod @@ -15,7 +15,6 @@ require ( k8s.io/apiserver v0.35.0 k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1 k8s.io/client-go v0.35.0 - k8s.io/component-base v0.35.0 k8s.io/klog/v2 v2.130.1 k8s.io/kubernetes v1.35.0 k8s.io/pod-security-admission v0.35.0 @@ -162,6 +161,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.35.0 // indirect k8s.io/cloud-provider v0.35.0 // indirect + k8s.io/component-base v0.35.0 // indirect k8s.io/component-helpers v0.35.0 // indirect k8s.io/controller-manager v0.35.0 // indirect k8s.io/cri-api v0.35.0 // indirect @@ -177,7 +177,7 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/vertical-pod-autoscaler/e2e/go.sum b/vertical-pod-autoscaler/e2e/go.sum index fc78eefc1db7..d49a91fcd8b0 100644 --- a/vertical-pod-autoscaler/e2e/go.sum +++ b/vertical-pod-autoscaler/e2e/go.sum @@ -492,7 +492,7 @@ sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/vertical-pod-autoscaler/e2e/integration/recommender.go b/vertical-pod-autoscaler/e2e/integration/recommender.go index 404bd359020e..367a4a72a106 100644 --- a/vertical-pod-autoscaler/e2e/integration/recommender.go +++ b/vertical-pod-autoscaler/e2e/integration/recommender.go @@ -96,7 +96,7 @@ func testIncludedAndIgnoredNamespaces(f *framework.Framework, vpaClientSet vpa_c ignoredVpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(ignoredNamespace). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). WithScalingMode(container1Name, vpa_types.ContainerScalingModeOff). WithContainer(container2Name). @@ -113,7 +113,7 @@ func testIncludedAndIgnoredNamespaces(f *framework.Framework, vpaClientSet vpa_c vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(includedNamespace). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). WithScalingMode(container1Name, vpa_types.ContainerScalingModeOff). WithContainer(container2Name). diff --git a/vertical-pod-autoscaler/e2e/utils/common.go b/vertical-pod-autoscaler/e2e/utils/common.go index 6481346e5c79..5c02934fcaf7 100644 --- a/vertical-pod-autoscaler/e2e/utils/common.go +++ b/vertical-pod-autoscaler/e2e/utils/common.go @@ -68,13 +68,6 @@ func init() { } } -// HamsterTargetRef is CrossVersionObjectReference of hamster app -var HamsterTargetRef = &autoscaling.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "hamster-deployment", -} - // RecommenderLabels are labels of VPA recommender var RecommenderLabels = map[string]string{"app": "vpa-recommender"} diff --git a/vertical-pod-autoscaler/e2e/v1/actuation.go b/vertical-pod-autoscaler/e2e/v1/actuation.go index 24f43264a992..75d407cad04f 100644 --- a/vertical-pod-autoscaler/e2e/v1/actuation.go +++ b/vertical-pod-autoscaler/e2e/v1/actuation.go @@ -69,7 +69,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate). WithContainer(containerName). AppendRecommendation( @@ -126,7 +126,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). WithContainer(container2Name). WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate). @@ -189,7 +189,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate). AppendRecommendation( @@ -231,7 +231,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate). AppendRecommendation( @@ -293,7 +293,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -325,7 +325,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithUpdateMode(vpa_types.UpdateModeOff). WithContainer(containerName). AppendRecommendation( @@ -363,7 +363,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithUpdateMode(vpa_types.UpdateModeInitial). WithContainer(containerName). AppendRecommendation( @@ -475,7 +475,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -534,7 +534,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -568,7 +568,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -605,7 +605,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). AppendRecommendation( test.Recommendation(). @@ -650,7 +650,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). AppendRecommendation( test.Recommendation(). @@ -715,7 +715,7 @@ var _ = ActuationSuiteE2eDescribe("Actuation", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). AppendRecommendation( test.Recommendation(). diff --git a/vertical-pod-autoscaler/e2e/v1/admission_controller.go b/vertical-pod-autoscaler/e2e/v1/admission_controller.go index 2fa12fdc4a6b..f1816ae5eed5 100644 --- a/vertical-pod-autoscaler/e2e/v1/admission_controller.go +++ b/vertical-pod-autoscaler/e2e/v1/admission_controller.go @@ -66,7 +66,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate). AppendRecommendation( @@ -99,7 +99,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -132,7 +132,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(removedContainerName). AppendRecommendation( test.Recommendation(). @@ -172,7 +172,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(removedContainerName). AppendRecommendation( test.Recommendation(). @@ -212,7 +212,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -248,7 +248,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). AppendRecommendation( test.Recommendation(). @@ -291,7 +291,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -341,7 +341,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -377,7 +377,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -426,7 +426,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -462,7 +462,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -498,7 +498,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). WithControlledValues(containerName, vpa_types.ContainerControlledValuesRequestsOnly). AppendRecommendation( @@ -539,7 +539,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -597,7 +597,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -646,7 +646,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). AppendRecommendation( test.Recommendation(). @@ -702,7 +702,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). AppendRecommendation( test.Recommendation(). @@ -754,7 +754,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -788,7 +788,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). AppendRecommendation( test.Recommendation(). @@ -822,7 +822,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). Get() @@ -846,7 +846,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). Get() diff --git a/vertical-pod-autoscaler/e2e/v1/recommender.go b/vertical-pod-autoscaler/e2e/v1/recommender.go index 58a8ff7ff16b..9c1029a8dd46 100644 --- a/vertical-pod-autoscaler/e2e/v1/recommender.go +++ b/vertical-pod-autoscaler/e2e/v1/recommender.go @@ -228,7 +228,7 @@ var _ = utils.RecommenderE2eDescribe("VPA CRD object", func() { vpaCRD = test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). Get() @@ -311,7 +311,7 @@ var _ = utils.RecommenderE2eDescribe("VPA CRD object", func() { vpaCRD2 := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). WithMinAllowed(containerName, "10000", ""). Get() @@ -338,7 +338,7 @@ var _ = utils.RecommenderE2eDescribe("VPA CRD object", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(containerName). WithMaxAllowed(containerName, "1m", ""). Get() @@ -382,7 +382,7 @@ var _ = utils.RecommenderE2eDescribe("VPA CRD object", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). WithContainer(container2Name). Get() @@ -406,7 +406,7 @@ var _ = utils.RecommenderE2eDescribe("VPA CRD object", func() { vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). - WithTargetRef(utils.HamsterTargetRef). + WithTargetRef(test.HamsterTargetRef). WithContainer(container1Name). WithScalingMode(container1Name, vpa_types.ContainerScalingModeOff). WithContainer(container2Name). diff --git a/vertical-pod-autoscaler/go.mod b/vertical-pod-autoscaler/go.mod index 8320bc654d4b..287da563458a 100644 --- a/vertical-pod-autoscaler/go.mod +++ b/vertical-pod-autoscaler/go.mod @@ -11,6 +11,7 @@ require ( github.com/prometheus/common v0.67.4 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 + go.uber.org/mock v0.6.0 golang.org/x/time v0.14.0 k8s.io/api v0.35.0 k8s.io/apimachinery v0.35.0 @@ -20,6 +21,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/metrics v0.35.0 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 + sigs.k8s.io/controller-runtime v0.23.1 ) exclude ( @@ -35,6 +37,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect @@ -67,7 +70,6 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/otel v1.39.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect - go.uber.org/mock v0.6.0 go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.30.0 // indirect @@ -82,11 +84,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.35.0 // indirect k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/vertical-pod-autoscaler/go.sum b/vertical-pod-autoscaler/go.sum index 0be62029f70c..451dc3996817 100644 --- a/vertical-pod-autoscaler/go.sum +++ b/vertical-pod-autoscaler/go.sum @@ -13,12 +13,16 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= @@ -129,6 +133,10 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -155,6 +163,8 @@ golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnps golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -168,6 +178,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJatA5T4= +k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU= k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= @@ -186,11 +198,13 @@ k8s.io/metrics v0.35.0 h1:xVFoqtAGm2dMNJAcB5TFZJPCen0uEqqNt52wW7ABbX8= k8s.io/metrics v0.35.0/go.mod h1:g2Up4dcBygZi2kQSEQVDByFs+VUwepJMzzQLJJLpq4M= k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.23.1 h1:TjJSM80Nf43Mg21+RCy3J70aj/W6KyvDtOlpKf+PupE= +sigs.k8s.io/controller-runtime v0.23.1/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/vertical-pod-autoscaler/hack/run-integration-tests.sh b/vertical-pod-autoscaler/hack/run-integration-tests.sh new file mode 100755 index 000000000000..003c82cfd470 --- /dev/null +++ b/vertical-pod-autoscaler/hack/run-integration-tests.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script sets up and runs the VPA integration tests using controller-runtime's envtest. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VPA_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +INTEGRATION_DIR="${VPA_DIR}/integration" + +# Kubernetes version to use for envtest binaries +# This should match the k8s.io/* dependency versions in go.mod +ENVTEST_K8S_VERSION="${ENVTEST_K8S_VERSION:-1.35.x}" + +# Directory to store envtest binaries +ENVTEST_ASSETS_DIR="${ENVTEST_ASSETS_DIR:-${HOME}/.local/share/kubebuilder-envtest}" + +echo "==> Setting up envtest environment..." + +# Check if setup-envtest is installed +if ! command -v setup-envtest &> /dev/null; then + echo "==> Installing setup-envtest..." + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +fi + +# Setup envtest binaries and get the path +echo "==> Downloading envtest binaries for Kubernetes ${ENVTEST_K8S_VERSION}..." +KUBEBUILDER_ASSETS="$(setup-envtest use "${ENVTEST_K8S_VERSION}" --bin-dir "${ENVTEST_ASSETS_DIR}" -p path)" +export KUBEBUILDER_ASSETS + +echo "==> Using envtest binaries from: ${KUBEBUILDER_ASSETS}" + +# Change to integration test directory +cd "${INTEGRATION_DIR}" + +# Run the tests +echo "==> Running integration tests..." +go test -tags=integration -v -timeout 300s -parallel 4 "$@" ./... diff --git a/vertical-pod-autoscaler/integration/main.go b/vertical-pod-autoscaler/integration/main.go new file mode 100644 index 000000000000..d71318f362a0 --- /dev/null +++ b/vertical-pod-autoscaler/integration/main.go @@ -0,0 +1,222 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "testing" + + appsv1 "k8s.io/api/apps/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/app" + recommender_config "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/config" +) + +// TestEnvironment holds the resources for a single test's environment. +// Each test gets its own isolated API server, clients, and kubeconfig. +type testEnvironment struct { + TestEnv *envtest.Environment + RestConfig *rest.Config + KubeClient *clientset.Clientset + VPAClient *vpa_clientset.Clientset + Kubeconfig string +} + +// Cleanup stops the test environment and removes temporary files. +func (te *testEnvironment) Cleanup() { + if te.Kubeconfig != "" { + _ = os.Remove(te.Kubeconfig) + } + if te.TestEnv != nil { + _ = te.TestEnv.Stop() + } +} + +// SetupTestEnvironment creates a new isolated envtest environment for a single test. +// This allows tests to run in parallel without interfering with each other. +// The caller should defer cleanup() to ensure proper resource cleanup. +func SetupTestEnvironment(t *testing.T) *testEnvironment { + t.Helper() + + // Get the path to the CRD YAML file relative to this test file + _, thisFile, _, _ := runtime.Caller(0) + crdPath := filepath.Join(filepath.Dir(thisFile), "..", "deploy", "vpa-v1-crd-gen.yaml") + + // envtest looks for binaries in the following order: + // 1. KUBEBUILDER_ASSETS environment variable + // 2. Default path: /usr/local/kubebuilder/bin + // To install the binaries, run: + // go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + // setup-envtest use --bin-dir /usr/local/kubebuilder/bin + // Or set KUBEBUILDER_ASSETS to point to the directory containing etcd and kube-apiserver + env := &envtest.Environment{ + CRDDirectoryPaths: []string{crdPath}, + ErrorIfCRDPathMissing: true, + } + + restConfig, err := env.Start() + if err != nil { + t.Fatalf("Failed to start envtest: %v. Make sure KUBEBUILDER_ASSETS is set or binaries are installed. "+ + "Run: go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest && "+ + "eval $(setup-envtest use -p env)", err) + } + + kubeClient, err := clientset.NewForConfig(restConfig) + if err != nil { + _ = env.Stop() + t.Fatalf("Failed to create kube client: %v", err) + } + + vpaClientConfig := rest.CopyConfig(restConfig) + vpaClientConfig.ContentType = "application/json" + vpaClient, err := vpa_clientset.NewForConfig(vpaClientConfig) + if err != nil { + _ = env.Stop() + t.Fatalf("Failed to create VPA client: %v", err) + } + + // Create a kubeconfig file for the recommender to use + kubeconfig := createKubeconfigFileForRestConfig(restConfig) + + return &testEnvironment{ + TestEnv: env, + RestConfig: restConfig, + KubeClient: kubeClient, + VPAClient: vpaClient, + Kubeconfig: kubeconfig, + } +} + +// getFreePort returns a free port number that can be used for the metrics server. +// This is necessary for parallel test execution to avoid port conflicts. +func getFreePort() (int, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer listener.Close() //nolint:errcheck + return listener.Addr().(*net.TCPAddr).Port, nil +} + +func createKubeconfigFileForRestConfig(restConfig *rest.Config) string { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + TLSServerName: restConfig.ServerName, + CertificateAuthorityData: restConfig.CAData, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + AuthInfo: "default-user", + } + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos["default-user"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + Token: restConfig.BearerToken, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authinfos, + } + kubeConfigFile, _ := os.CreateTemp("", "kubeconfig") + _ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name()) + return kubeConfigFile.Name() +} + +// StartRecommender creates and starts a recommender app with the given config. +// It returns a context for the recommender and a cancel function that should be deferred. +// The recommender runs in a background goroutine. +func StartRecommender(t *testing.T, config *recommender_config.RecommenderConfig) (recommenderCtx context.Context, cancel func()) { + t.Helper() + + // Get a free port for the metrics server to avoid conflicts in parallel tests + port, err := getFreePort() + if err != nil { + t.Fatalf("Failed to get free port for metrics: %v", err) + } + config.Address = fmt.Sprintf(":%d", port) + + recommenderApp, err := app.NewRecommenderApp(config) + if err != nil { + t.Fatalf("Failed to create recommender app: %v", err) + } + + recommenderCtx, recommenderCancel := context.WithCancel(context.Background()) + + // Start the recommender in a goroutine + errChan := make(chan error, 1) + doneChan := make(chan struct{}) + leaderElection := app.DefaultLeaderElectionConfiguration() + leaderElection.LeaderElect = false // Disable leader election for testing + + go func() { + defer close(doneChan) + t.Logf("Starting recommender app on port %d...", port) + err := recommenderApp.Run(recommenderCtx, leaderElection) + if err != nil && recommenderCtx.Err() == nil { + errChan <- err + } + }() + + return recommenderCtx, recommenderCancel +} + +// NewHamsterDeployment creates a simple hamster deployment for testing. +func NewHamsterDeployment(ns string, replicas int32, labels map[string]string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hamster-deployment", + Namespace: ns, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: apiv1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: apiv1.PodSpec{ + Containers: []apiv1.Container{{ + Name: "hamster", + Image: "busybox", + }}, + }, + }, + }, + } +} diff --git a/vertical-pod-autoscaler/integration/recommender/recommender_test.go b/vertical-pod-autoscaler/integration/recommender/recommender_test.go new file mode 100644 index 000000000000..a9cb9482d38c --- /dev/null +++ b/vertical-pod-autoscaler/integration/recommender/recommender_test.go @@ -0,0 +1,423 @@ +//go:build integration +// +build integration + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommender + +import ( + "context" + "testing" + "time" + + apiv1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + "k8s.io/autoscaler/vertical-pod-autoscaler/common" + "k8s.io/autoscaler/vertical-pod-autoscaler/integration" + recommender_config "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/config" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" +) + +func TestRecommenderWithNamespaceFiltering(t *testing.T) { + t.Parallel() + + // Setup isolated test environment + env := integration.SetupTestEnvironment(t) + defer env.Cleanup() + + ctx := t.Context() + + // Create test namespaces + watchedNS := "ns-filtering-watched" + ignoredNS := "ns-filtering-ignored" + + for _, ns := range []string{watchedNS, ignoredNS} { + _, err := env.KubeClient.CoreV1().Namespaces().Create(ctx, &apiv1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create namespace %s: %v", ns, err) + } + defer func(ns string) { + _ = env.KubeClient.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + }(ns) + } + + // Create VPA objects in both namespaces + for _, ns := range []string{watchedNS, ignoredNS} { + vpa := test.VerticalPodAutoscaler(). + WithName("test-vpa"). + WithContainer("hamster"). + WithNamespace(ns). + WithTargetRef(test.HamsterTargetRef). + Get() + + _, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Create(ctx, vpa, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create VPA in namespace %s: %v", ns, err) + } + } + + // Configure the recommender to watch only the watched namespace + config := recommender_config.DefaultRecommenderConfig() + config.CommonFlags = &common.CommonFlags{ + KubeConfig: env.Kubeconfig, + VpaObjectNamespace: watchedNS, // Only watch the watched namespace + IgnoredVpaObjectNamespaces: "", + } + config.MetricsFetcherInterval = 1 * time.Second // Short interval for testing + + _, cancel := integration.StartRecommender(t, config) + defer cancel() + + // Wait for the recommender to process the VPA in the watched namespace. + // The recommender should add status conditions to VPAs it manages. + err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (done bool, err error) { + watchedVPA, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(watchedNS).Get(ctx, "test-vpa", metav1.GetOptions{}) + if err != nil { + return false, err + } + // watched namespace should have status updates + if len(watchedVPA.Status.Conditions) > 0 { + return true, nil + } + return false, nil + }) + if err != nil { + t.Fatalf("VPA in watched namespace should have status conditions: %v", err) + } + + // Fetch VPA in the ignored namespace. + // The recommender should NOT have added a status conditions to this VPA. + ignoredVPA, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ignoredNS).Get(ctx, "test-vpa", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unable to get VPA in ignored namespace: %v", err) + } + + if len(ignoredVPA.Status.Conditions) != 0 { + t.Fatal("VPA in ignored namespace should NOT have status conditions") + } +} + +// TestRecommenderWithNamespaceExclusions verifies that the recommender +// ignores namespaces that it's configured to ignore, and watches VPAs in +// all other namespaces. +func TestRecommenderWithNamespaceExclusions(t *testing.T) { + t.Parallel() + + // Setup isolated test environment + env := integration.SetupTestEnvironment(t) + defer env.Cleanup() + + ctx := t.Context() + + // Create test namespaces + watchedNS := "ns-exclusions-watched" + ignoredNS := "ns-exclusions-ignored" + + for _, ns := range []string{watchedNS, ignoredNS} { + _, err := env.KubeClient.CoreV1().Namespaces().Create(ctx, &apiv1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create namespace %s: %v", ns, err) + } + defer func(ns string) { + _ = env.KubeClient.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + }(ns) + } + + // Create VPA objects in both namespaces + for _, ns := range []string{watchedNS, ignoredNS} { + vpa := test.VerticalPodAutoscaler(). + WithName("test-vpa"). + WithContainer("hamster"). + WithNamespace(ns). + WithTargetRef(test.HamsterTargetRef). + Get() + + _, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Create(ctx, vpa, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create VPA in namespace %s: %v", ns, err) + } + } + + // Configure the recommender to exclude the ignored namespace + config := recommender_config.DefaultRecommenderConfig() + config.CommonFlags = &common.CommonFlags{ + KubeConfig: env.Kubeconfig, + VpaObjectNamespace: "", // Watch all namespaces + IgnoredVpaObjectNamespaces: ignoredNS, + } + config.MetricsFetcherInterval = 1 * time.Second // Short interval for testing + + _, cancel := integration.StartRecommender(t, config) + defer cancel() + + // Wait for the recommender to process the VPA in the watched namespace. + // The recommender should add status conditions to VPAs it manages. + err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (done bool, err error) { + watchedVPA, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(watchedNS).Get(ctx, "test-vpa", metav1.GetOptions{}) + if err != nil { + return false, err + } + // watched namespace should have status updates + if len(watchedVPA.Status.Conditions) > 0 { + return true, nil + } + return false, nil + }) + if err != nil { + t.Fatalf("VPA in watched namespace should have status conditions: %v", err) + } + + // Fetch VPA in the ignored namespace. + // The recommender should NOT have added a status conditions to this VPA. + ignoredVPA, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ignoredNS).Get(ctx, "test-vpa", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unable to get VPA in ignored namespace: %v", err) + } + + if len(ignoredVPA.Status.Conditions) != 0 { + t.Fatal("VPA in ignored namespace should NOT have status conditions") + } +} + +// TestCRDCheckpointGC verifies that the recommender's checkpoint garbage collection +// correctly deletes checkpoints for VPAs that have been deleted +func TestCRDCheckpointGC(t *testing.T) { + t.Parallel() + + // Setup isolated test environment + env := integration.SetupTestEnvironment(t) + defer env.Cleanup() + + ctx := t.Context() + + ns := "checkpoint-gc-test" + + // Create test namespace + _, err := env.KubeClient.CoreV1().Namespaces().Create(ctx, &apiv1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create namespace %s: %v", ns, err) + } + defer func(ns string) { + _ = env.KubeClient.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + }(ns) + + // Create a Deployment that the VPA will target + deploymentLabel := map[string]string{"app": "hamster"} + deployment := integration.NewHamsterDeployment(ns, 1, deploymentLabel) + + _, err = env.KubeClient.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create Deployment in namespace %s: %v", ns, err) + } + + // Create Pods matching the deployment (simulating what the deployment controller would do) + pod := test.Pod(). + WithName("hamster-pod-0"). + WithLabels(deploymentLabel). + WithPhase(apiv1.PodRunning). + AddContainer(test.Container(). + WithName("hamster"). + WithImage("busybox"). + WithCPURequest(resource.MustParse("100m")). + WithMemRequest(resource.MustParse("50Mi")). + Get()). + AddContainerStatus(apiv1.ContainerStatus{ + Name: "hamster", + Ready: true, + State: apiv1.ContainerState{ + Running: &apiv1.ContainerStateRunning{ + StartedAt: metav1.Now(), + }, + }, + }). + Get() + pod.Namespace = ns + + createdPod, err := env.KubeClient.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create Pod in namespace %s: %v", ns, err) + } + + // Since the API server ignores status on Create, we need to update it separately + createdPod.Status = pod.Status + _, err = env.KubeClient.CoreV1().Pods(ns).UpdateStatus(ctx, createdPod, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update Pod status in namespace %s: %v", ns, err) + } + + // Create the VPA targeting the deployment + vpa := test.VerticalPodAutoscaler(). + WithName("test-vpa"). + WithNamespace(ns). + WithContainer("hamster"). + WithTargetRef(test.HamsterTargetRef). + Get() + + _, err = env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Create(ctx, vpa, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create VPA in namespace %s: %v", ns, err) + } + + config := recommender_config.DefaultRecommenderConfig() + config.CommonFlags = &common.CommonFlags{ + KubeConfig: env.Kubeconfig, + } + config.MetricsFetcherInterval = 1 * time.Second // Short interval for testing + config.CheckpointsGCInterval = 1 * time.Second // Short interval for testing + + _, cancel := integration.StartRecommender(t, config) + defer cancel() + + // Poll until the VPA has created a checkpoint for the VPA + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (done bool, err error) { + _, err = env.VPAClient.AutoscalingV1().VerticalPodAutoscalerCheckpoints(ns).Get(ctx, "test-vpa-hamster", metav1.GetOptions{}) + if err == nil { + return true, nil // Checkpoint found + } + if apierrors.IsNotFound(err) { + return false, nil // Not found yet, keep polling + } + return false, err // Real error, stop and fail + }) + + if err != nil { + t.Fatalf("Timed out waiting for checkpoint to be created: %v", err) + } + + // Delete the VPA, which should trigger deletion of the checkpoint by the GC + err = env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Delete(ctx, "test-vpa", metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Failed to delete VPA: %v", err) + } + + // Poll until the VPA has deleted a checkpoint for the VPA + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 300*time.Second, true, func(ctx context.Context) (done bool, err error) { + cp, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalerCheckpoints(ns).Get(ctx, "test-vpa-hamster", metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil // Checkpoint was deleted by GC - this is what we're testing for + } + return false, err // Some other error (e.g., server shutdown) - fail the test + } + t.Log("Found Checkpoint, still waiting for GC", cp.Name) + return false, nil + }) + + if err != nil { + t.Fatalf("Timed out waiting for VPA Checkpoint to be garbage collected: %v", err) + } +} + +// Test RecomenderName tests that a recommender only processes VPAs +// that specify its recommender name, and ignores VPAs targeting other recommenders. +func TestRecommenderName(t *testing.T) { + t.Parallel() + + // Setup isolated test environment + env := integration.SetupTestEnvironment(t) + defer env.Cleanup() + + ctx := t.Context() + + ns := "recommender-name-test" + + // Create test namespace + _, err := env.KubeClient.CoreV1().Namespaces().Create(ctx, &apiv1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create namespace %s: %v", ns, err) + } + defer func(ns string) { + _ = env.KubeClient.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + }(ns) + + // Create two VPAs that target different recommenders: + // - "vpa-for-custom-recommender" uses recommender "custom-recommender" + // - "vpa-for-default-recommender" uses recommender "default" (empty string means default) + vpaCustom := test.VerticalPodAutoscaler(). + WithName("vpa-for-custom-recommender"). + WithContainer("hamster"). + WithNamespace(ns). + WithRecommender("custom-recommender"). + WithTargetRef(test.HamsterTargetRef). + Get() + + vpaDefault := test.VerticalPodAutoscaler(). + WithName("vpa-for-default-recommender"). + WithContainer("hamster"). + WithNamespace(ns). + // No WithRecommender = uses default recommender + WithTargetRef(test.HamsterTargetRef). + Get() + + _, err = env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Create(ctx, vpaCustom, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create VPA: %v", err) + } + + _, err = env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Create(ctx, vpaDefault, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create VPA: %v", err) + } + + // Start a recommender named "custom-recommender" + // It should only process VPAs that specify this recommender name + config := recommender_config.DefaultRecommenderConfig() + config.CommonFlags = &common.CommonFlags{ + KubeConfig: env.Kubeconfig, + } + config.MetricsFetcherInterval = 1 * time.Second + config.RecommenderName = "custom-recommender" + + _, cancel := integration.StartRecommender(t, config) + defer cancel() + + // The VPA targeting "custom-recommender" should get status updates + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (done bool, err error) { + vpa, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Get(ctx, "vpa-for-custom-recommender", metav1.GetOptions{}) + if err != nil { + return false, err + } + if len(vpa.Status.Conditions) > 0 { + return true, nil + } + return false, nil + }) + if err != nil { + t.Fatalf("VPA targeting custom-recommender should have status conditions: %v", err) + } + + vpa, err := env.VPAClient.AutoscalingV1().VerticalPodAutoscalers(ns).Get(ctx, "vpa-for-default-recommender", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unable to get VPA for the default recommender: %v", err) + } + // We expect NO conditions - if we see any, the test should fail + if len(vpa.Status.Conditions) > 0 { + t.Fatal("VPA targeting default recommender should NOT have status conditions (custom-recommender should ignore it)") + } +} diff --git a/vertical-pod-autoscaler/pkg/admission-controller/certs.go b/vertical-pod-autoscaler/pkg/admission-controller/certs.go index 2fe64d88bb30..d7c6d32b2491 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/certs.go +++ b/vertical-pod-autoscaler/pkg/admission-controller/certs.go @@ -32,9 +32,10 @@ import ( "k8s.io/klog/v2" ) -type certsConfig struct { - clientCaFile, tlsCertFile, tlsPrivateKey *string - reload *bool +// CertsConfig holds configuration related to TLS certificates +type CertsConfig struct { + ClientCaFile, TlsCertFile, TlsPrivateKey *string + Reload *bool } func readFile(filePath string) []byte { diff --git a/vertical-pod-autoscaler/pkg/admission-controller/config.go b/vertical-pod-autoscaler/pkg/admission-controller/config.go index dc89fa268017..bda6b58f67f9 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/config.go +++ b/vertical-pod-autoscaler/pkg/admission-controller/config.go @@ -28,6 +28,8 @@ import ( "k8s.io/client-go/kubernetes" typedadmregv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1" "k8s.io/klog/v2" + + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/config" ) const ( @@ -36,7 +38,7 @@ const ( ) // MutatingWebhookConfigurationInterface -func configTLS(cfg certsConfig, minTlsVersion, ciphers string, stop <-chan struct{}, mutatingWebhookClient typedadmregv1.MutatingWebhookConfigurationInterface) *tls.Config { +func configTLS(cfg config.CertsConfig, minTlsVersion, ciphers string, stop <-chan struct{}, mutatingWebhookClient typedadmregv1.MutatingWebhookConfigurationInterface) *tls.Config { var tlsVersion uint16 var ciphersuites []uint16 reverseCipherMap := make(map[string]uint16) @@ -67,11 +69,11 @@ func configTLS(cfg certsConfig, minTlsVersion, ciphers string, stop <-chan struc MinVersion: tlsVersion, CipherSuites: ciphersuites, } - if *cfg.reload { + if cfg.Reload { cr := certReloader{ - tlsCertPath: *cfg.tlsCertFile, - tlsKeyPath: *cfg.tlsPrivateKey, - clientCaPath: *cfg.clientCaFile, + tlsCertPath: cfg.TlsCertFile, + tlsKeyPath: cfg.TlsPrivateKey, + clientCaPath: cfg.ClientCaFile, mutatingWebhookClient: mutatingWebhookClient, } if err := cr.load(); err != nil { @@ -82,7 +84,7 @@ func configTLS(cfg certsConfig, minTlsVersion, ciphers string, stop <-chan struc } config.GetCertificate = cr.getCertificate } else { - cert, err := tls.LoadX509KeyPair(*cfg.tlsCertFile, *cfg.tlsPrivateKey) + cert, err := tls.LoadX509KeyPair(cfg.TlsCertFile, cfg.TlsPrivateKey) if err != nil { klog.Fatal(err) } diff --git a/vertical-pod-autoscaler/pkg/admission-controller/config/config.go b/vertical-pod-autoscaler/pkg/admission-controller/config/config.go new file mode 100644 index 000000000000..e140173c8a70 --- /dev/null +++ b/vertical-pod-autoscaler/pkg/admission-controller/config/config.go @@ -0,0 +1,116 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "flag" + "os" + + "k8s.io/klog/v2" + + "k8s.io/autoscaler/vertical-pod-autoscaler/common" +) + +// CertsConfig holds configuration related to TLS certificates +type CertsConfig struct { + ClientCaFile string + TlsCertFile string + TlsPrivateKey string + Reload bool +} + +// AdmissionControllerConfig holds all configuration for the admission controller component +type AdmissionControllerConfig struct { + // Common flags + CommonFlags *common.CommonFlags + + CertsConfiguration *CertsConfig + + Ciphers string + MinTlsVersion string + Port int + Address string + Namespace string + ServiceName string + WebhookAddress string + WebhookPort string + WebhookTimeout int + WebhookFailurePolicy bool + RegisterWebhook bool + WebhookLabels string + RegisterByURL bool +} + +// DefaultAdmissionControllerConfig returns a AdmissionControllerConfig with default values +func DefaultAdmissionControllerConfig() *AdmissionControllerConfig { + return &AdmissionControllerConfig{ + CommonFlags: common.DefaultCommonConfig(), + CertsConfiguration: &CertsConfig{ + ClientCaFile: "/etc/tls-certs/caCert.pem", + TlsCertFile: "/etc/tls-certs/serverCert.pem", + TlsPrivateKey: "/etc/tls-certs/serverKey.pem", + Reload: false, + }, + Ciphers: "", + MinTlsVersion: "tls1_2", + Port: 8000, + Address: ":8944", + Namespace: os.Getenv("NAMESPACE"), + ServiceName: "vpa-webhook", + WebhookAddress: "", + WebhookPort: "", + WebhookTimeout: 30, + WebhookFailurePolicy: false, + RegisterWebhook: true, + WebhookLabels: "", + RegisterByURL: false, + } +} + +// InitAdmissionControllerFlags initializes the flags for the admission controller component +func InitAdmissionControllerFlags() *AdmissionControllerConfig { + config := DefaultAdmissionControllerConfig() + config.CommonFlags = common.InitCommonFlags() + + flag.StringVar(&config.CertsConfiguration.ClientCaFile, "client-ca-file", config.CertsConfiguration.ClientCaFile, "Path to CA PEM file.") + flag.StringVar(&config.CertsConfiguration.TlsCertFile, "tls-cert-file", config.CertsConfiguration.TlsCertFile, "Path to server certificate PEM file.") + flag.StringVar(&config.CertsConfiguration.TlsPrivateKey, "tls-private-key", config.CertsConfiguration.TlsPrivateKey, "Path to server certificate key PEM file.") + flag.BoolVar(&config.CertsConfiguration.Reload, "reload-cert", config.CertsConfiguration.Reload, "If set to true, reload leaf and CA certificates when changed.") + + flag.StringVar(&config.Ciphers, "tls-ciphers", config.Ciphers, "A comma-separated or colon-separated list of ciphers to accept. Only works when min-tls-version is set to tls1_2.") + flag.StringVar(&config.MinTlsVersion, "min-tls-version", config.MinTlsVersion, "The minimum TLS version to accept. Must be set to either tls1_2 (default) or tls1_3.") + flag.IntVar(&config.Port, "port", config.Port, "The port to listen on.") + flag.StringVar(&config.Address, "address", config.Address, "The address to expose Prometheus metrics.") + flag.StringVar(&config.ServiceName, "webhook-service", config.ServiceName, "Kubernetes service under which webhook is registered. Used when registerByURL is set to false.") + flag.StringVar(&config.WebhookAddress, "webhook-address", config.WebhookAddress, "Address under which webhook is registered. Used when registerByURL is set to true.") + flag.StringVar(&config.WebhookPort, "webhook-port", config.WebhookPort, "Server Port for Webhook") + flag.IntVar(&config.WebhookTimeout, "webhook-timeout-seconds", config.WebhookTimeout, "Timeout in seconds that the API server should wait for this webhook to respond before failing.") + flag.BoolVar(&config.WebhookFailurePolicy, "webhook-failure-policy-fail", config.WebhookFailurePolicy, "If set to true, will configure the admission webhook failurePolicy to \"Fail\". Use with caution.") + flag.BoolVar(&config.RegisterWebhook, "register-webhook", config.RegisterWebhook, "If set to true, admission webhook object will be created on start up to register with the API server.") + flag.StringVar(&config.WebhookLabels, "webhook-labels", config.WebhookLabels, "Comma separated list of labels to add to the webhook object. Format: key1:value1,key2:value2") + flag.BoolVar(&config.RegisterByURL, "register-by-url", config.RegisterByURL, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name") + + return config +} + +// ValidateAdmissionControllerConfig performs validation of the admission-controller flags +func ValidateAdmissionControllerConfig(config *AdmissionControllerConfig) { + if len(config.CommonFlags.VpaObjectNamespace) > 0 && len(config.CommonFlags.IgnoredVpaObjectNamespaces) > 0 { + klog.ErrorS(nil, "--vpa-object-namespace and --ignored-vpa-object-namespaces are mutually exclusive and can't be set together.") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } +} diff --git a/vertical-pod-autoscaler/pkg/admission-controller/main.go b/vertical-pod-autoscaler/pkg/admission-controller/main.go index e46a7c465655..49bfe0680a4e 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/main.go +++ b/vertical-pod-autoscaler/pkg/admission-controller/main.go @@ -17,7 +17,7 @@ limitations under the License. package main import ( - "flag" + "context" "fmt" "net/http" "os" @@ -32,6 +32,7 @@ import ( "k8s.io/klog/v2" "k8s.io/autoscaler/vertical-pod-autoscaler/common" + admissioncontroller_config "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/config" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/logic" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/pod" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/pod/patch" @@ -58,53 +59,33 @@ const ( webHookDelay = 10 * time.Second ) -var ( - certsConfiguration = &certsConfig{ - clientCaFile: flag.String("client-ca-file", "/etc/tls-certs/caCert.pem", "Path to CA PEM file."), - tlsCertFile: flag.String("tls-cert-file", "/etc/tls-certs/serverCert.pem", "Path to server certificate PEM file."), - tlsPrivateKey: flag.String("tls-private-key", "/etc/tls-certs/serverKey.pem", "Path to server certificate key PEM file."), - reload: flag.Bool("reload-cert", false, "If set to true, reload leaf and CA certificates when changed."), - } - ciphers = flag.String("tls-ciphers", "", "A comma-separated or colon-separated list of ciphers to accept. Only works when min-tls-version is set to tls1_2.") - minTlsVersion = flag.String("min-tls-version", "tls1_2", "The minimum TLS version to accept. Must be set to either tls1_2 (default) or tls1_3.") - port = flag.Int("port", 8000, "The port to listen on.") - address = flag.String("address", ":8944", "The address to expose Prometheus metrics.") - namespace = os.Getenv("NAMESPACE") - serviceName = flag.String("webhook-service", "vpa-webhook", "Kubernetes service under which webhook is registered. Used when registerByURL is set to false.") - webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.") - webhookPort = flag.String("webhook-port", "", "Server Port for Webhook") - webhookTimeout = flag.Int("webhook-timeout-seconds", 30, "Timeout in seconds that the API server should wait for this webhook to respond before failing.") - webHookFailurePolicy = flag.Bool("webhook-failure-policy-fail", false, "If set to true, will configure the admission webhook failurePolicy to \"Fail\". Use with caution.") - registerWebhook = flag.Bool("register-webhook", true, "If set to true, admission webhook object will be created on start up to register with the API server.") - webhookLabels = flag.String("webhook-labels", "", "Comma separated list of labels to add to the webhook object. Format: key1:value1,key2:value2") - registerByURL = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name") -) - func main() { - commonFlags := common.InitCommonFlags() + config := admissioncontroller_config.InitAdmissionControllerFlags() + admissioncontroller_config.ValidateAdmissionControllerConfig(config) + common.ValidateCommonConfig(config.CommonFlags) + klog.InitFlags(nil) common.InitLoggingFlags() features.MutableFeatureGate.AddFlag(pflag.CommandLine) kube_flag.InitFlags() klog.V(1).InfoS("Starting Vertical Pod Autoscaler Admission Controller", "version", common.VerticalPodAutoscalerVersion()) - if len(commonFlags.VpaObjectNamespace) > 0 && len(commonFlags.IgnoredVpaObjectNamespaces) > 0 { - klog.ErrorS(nil, "--vpa-object-namespace and --ignored-vpa-object-namespaces are mutually exclusive and can't be set together.") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } - healthCheck := metrics.NewHealthCheck(time.Minute) metrics_admission.Register() - server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) + server.Initialize(&config.CommonFlags.EnableProfiling, healthCheck, &config.Address) + + kubeConfig := common.CreateKubeConfigOrDie(config.CommonFlags.KubeConfig, float32(config.CommonFlags.KubeApiQps), int(config.CommonFlags.KubeApiBurst)) + + ctx := context.Background() + + vpaClient := vpa_clientset.NewForConfigOrDie(kubeConfig) + vpaLister := vpa_api_util.NewVpasLister(vpaClient, make(chan struct{}), config.CommonFlags.VpaObjectNamespace) + kubeClient := kube_client.NewForConfigOrDie(kubeConfig) - config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) + factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(config.CommonFlags.VpaObjectNamespace)) + targetSelectorFetcher := target.NewVpaTargetSelectorFetcher(ctx, kubeConfig, kubeClient, factory) + controllerFetcher := controllerfetcher.NewControllerFetcher(ctx, kubeConfig, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) - vpaClient := vpa_clientset.NewForConfigOrDie(config) - vpaLister := vpa_api_util.NewVpasLister(vpaClient, make(chan struct{}), commonFlags.VpaObjectNamespace) - kubeClient := kube_client.NewForConfigOrDie(config) - factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(commonFlags.VpaObjectNamespace)) - targetSelectorFetcher := target.NewVpaTargetSelectorFetcher(config, kubeClient, factory) - controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) podPreprocessor := pod.NewDefaultPreProcessor() vpaPreprocessor := vpa.NewDefaultPreProcessor() var limitRangeCalculator limitrange.LimitRangeCalculator @@ -134,8 +115,8 @@ func main() { } statusNamespace := status.AdmissionControllerStatusNamespace - if namespace != "" { - statusNamespace = namespace + if config.Namespace != "" { + statusNamespace = config.Namespace } statusUpdater := status.NewUpdater( kubeClient, @@ -152,30 +133,30 @@ func main() { healthCheck.UpdateLastActivity() }) var mutatingWebhookClient typedadmregv1.MutatingWebhookConfigurationInterface - if *registerWebhook { + if config.RegisterWebhook { mutatingWebhookClient = kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations() } server := &http.Server{ - Addr: fmt.Sprintf(":%d", *port), - TLSConfig: configTLS(*certsConfiguration, *minTlsVersion, *ciphers, stopCh, mutatingWebhookClient), + Addr: fmt.Sprintf(":%d", config.Port), + TLSConfig: configTLS(*config.CertsConfiguration, config.MinTlsVersion, config.Ciphers, stopCh, mutatingWebhookClient), } - url := fmt.Sprintf("%v:%v", *webhookAddress, *webhookPort) - ignoredNamespaces := strings.Split(commonFlags.IgnoredVpaObjectNamespaces, ",") + url := fmt.Sprintf("%v:%v", config.WebhookAddress, config.WebhookPort) + ignoredNamespaces := strings.Split(config.CommonFlags.IgnoredVpaObjectNamespaces, ",") go func() { - if *registerWebhook { + if config.RegisterWebhook { selfRegistration( kubeClient, - readFile(*certsConfiguration.clientCaFile), + readFile(config.CertsConfiguration.ClientCaFile), webHookDelay, - namespace, - *serviceName, + config.Namespace, + config.ServiceName, url, - *registerByURL, - int32(*webhookTimeout), - commonFlags.VpaObjectNamespace, + config.RegisterByURL, + int32(config.WebhookTimeout), + config.CommonFlags.VpaObjectNamespace, ignoredNamespaces, - *webHookFailurePolicy, - *webhookLabels, + config.WebhookFailurePolicy, + config.WebhookLabels, ) } // Start status updates after the webhook is initialized. diff --git a/vertical-pod-autoscaler/pkg/recommender/app/app.go b/vertical-pod-autoscaler/pkg/recommender/app/app.go new file mode 100644 index 000000000000..b606cd698bfb --- /dev/null +++ b/vertical-pod-autoscaler/pkg/recommender/app/app.go @@ -0,0 +1,326 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/informers" + kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + componentbaseconfig "k8s.io/component-base/config" + "k8s.io/klog/v2" + resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" + + "k8s.io/autoscaler/vertical-pod-autoscaler/common" + vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/checkpoint" + recommender_config "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/config" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" + input_metrics "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/metrics" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/routines" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" + metrics_quality "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/quality" + metrics_recommender "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/recommender" + metrics_resources "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/resources" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" +) + +const ( + // aggregateContainerStateGCInterval defines how often expired AggregateContainerStates are garbage collected. + aggregateContainerStateGCInterval = 1 * time.Hour + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. + scaleCacheLoopPeriod = 7 * time.Second + defaultResyncPeriod time.Duration = 10 * time.Minute +) + +// RecommenderApp represents the recommender application +type RecommenderApp struct { + config *recommender_config.RecommenderConfig +} + +// NewRecommenderApp creates a new RecommenderApp with the given configuration +func NewRecommenderApp(config *recommender_config.RecommenderConfig) (*RecommenderApp, error) { + if config == nil { + return nil, errors.New("config cannot be nil") + } + + // Load bearer token from file if specified + if config.PrometheusBearerTokenFile != "" { + fileContent, err := os.ReadFile(config.PrometheusBearerTokenFile) + if err != nil { + return nil, fmt.Errorf("unable to read bearer token file %s: %w", config.PrometheusBearerTokenFile, err) + } + config.PrometheusBearerToken = strings.TrimSpace(string(fileContent)) + } + + return &RecommenderApp{ + config: config, + }, nil +} + +// Run starts the recommender with the given context +func (app *RecommenderApp) Run(ctx context.Context, leaderElection componentbaseconfig.LeaderElectionConfiguration) error { + stopCh := make(chan struct{}) + // Close stopCh when context is canceled to signal all goroutines to stop + go func() { + <-ctx.Done() + close(stopCh) + }() + + healthCheck := metrics.NewHealthCheck(app.config.MetricsFetcherInterval * 5) + metrics_recommender.Register() + metrics_quality.Register() + metrics_resources.Register() + server.InitializeWithContext(ctx, &app.config.CommonFlags.EnableProfiling, healthCheck, &app.config.Address) + + if !leaderElection.LeaderElect { + return app.run(ctx, stopCh, healthCheck) + } + id, err := os.Hostname() + if err != nil { + return fmt.Errorf("unable to get hostname: %w", err) + } + + id = id + "_" + string(uuid.NewUUID()) + + config := common.CreateKubeConfigOrDie(app.config.CommonFlags.KubeConfig, float32(app.config.CommonFlags.KubeApiQps), int(app.config.CommonFlags.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + + lock, err := resourcelock.New( + leaderElection.ResourceLock, + leaderElection.ResourceNamespace, + leaderElection.ResourceName, + kubeClient.CoreV1(), + kubeClient.CoordinationV1(), + resourcelock.ResourceLockConfig{ + Identity: id, + }, + ) + if err != nil { + return fmt.Errorf("unable to create leader election lock: %w", err) + } + + leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaderElection.LeaseDuration.Duration, + RenewDeadline: leaderElection.RenewDeadline.Duration, + RetryPeriod: leaderElection.RetryPeriod.Duration, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + if err := app.run(ctx, stopCh, healthCheck); err != nil { + klog.Fatalf("Error running recommender: %v", err) + } + }, + OnStoppedLeading: func() { + klog.Fatal("lost master") + }, + }, + }) + + return nil +} + +func (app *RecommenderApp) run(ctx context.Context, stopCh chan struct{}, healthCheck *metrics.HealthCheck) error { + config := common.CreateKubeConfigOrDie(app.config.CommonFlags.KubeConfig, float32(app.config.CommonFlags.KubeApiQps), int(app.config.CommonFlags.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + clusterState := model.NewClusterState(aggregateContainerStateGCInterval) + factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(app.config.CommonFlags.VpaObjectNamespace)) + controllerFetcher := controllerfetcher.NewControllerFetcher(ctx, config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + podLister, oomObserver := input.NewPodListerAndOOMObserver(ctx, kubeClient, app.config.CommonFlags.VpaObjectNamespace, stopCh) + + factory.Start(stopCh) + informerMap := factory.WaitForCacheSync(stopCh) + for kind, synced := range informerMap { + if !synced { + return fmt.Errorf("could not sync cache for the %s informer", kind.String()) + } + } + + model.InitializeAggregationsConfig(model.NewAggregationsConfig( + app.config.MemoryAggregationInterval, + app.config.MemoryAggregationIntervalCount, + app.config.MemoryHistogramDecayHalfLife, + app.config.CpuHistogramDecayHalfLife, + app.config.OOMBumpUpRatio, + app.config.OOMMinBumpUp, + )) + + useCheckpoints := app.config.Storage != "prometheus" + + var postProcessors []routines.RecommendationPostProcessor + if app.config.PostProcessorCPUasInteger { + postProcessors = append(postProcessors, &routines.IntegerCPUPostProcessor{}) + } + + globalMaxAllowed := app.initGlobalMaxAllowed() + // CappingPostProcessor, should always come in the last position for post-processing + postProcessors = append(postProcessors, routines.NewCappingRecommendationProcessor(globalMaxAllowed)) + + var source input_metrics.PodMetricsLister + if app.config.UseExternalMetrics { + resourceMetrics := map[apiv1.ResourceName]string{} + if app.config.ExternalCpuMetric != "" { + resourceMetrics[apiv1.ResourceCPU] = app.config.ExternalCpuMetric + } + if app.config.ExternalMemoryMetric != "" { + resourceMetrics[apiv1.ResourceMemory] = app.config.ExternalMemoryMetric + } + externalClientOptions := &input_metrics.ExternalClientOptions{ + ResourceMetrics: resourceMetrics, + ContainerNameLabel: app.config.CtrNameLabel, + } + klog.V(1).InfoS("Using External Metrics", "options", externalClientOptions) + source = input_metrics.NewExternalClient(config, clusterState, *externalClientOptions) + } else { + klog.V(1).InfoS("Using Metrics Server") + source = input_metrics.NewPodMetricsesSource(resourceclient.NewForConfigOrDie(config)) + } + + ignoredNamespaces := strings.Split(app.config.CommonFlags.IgnoredVpaObjectNamespaces, ",") + + clusterStateFeeder := input.ClusterStateFeederFactory{ + PodLister: podLister, + OOMObserver: oomObserver, + KubeClient: kubeClient, + MetricsClient: input_metrics.NewMetricsClient(source, app.config.CommonFlags.VpaObjectNamespace, "default-metrics-client"), + VpaCheckpointClient: vpa_clientset.NewForConfigOrDie(config).AutoscalingV1(), + VpaLister: vpa_api_util.NewVpasLister(vpa_clientset.NewForConfigOrDie(config), stopCh, app.config.CommonFlags.VpaObjectNamespace), + VpaCheckpointLister: vpa_api_util.NewVpaCheckpointLister(vpa_clientset.NewForConfigOrDie(config), stopCh, app.config.CommonFlags.VpaObjectNamespace), + ClusterState: clusterState, + SelectorFetcher: target.NewVpaTargetSelectorFetcher(ctx, config, kubeClient, factory), + MemorySaveMode: app.config.MemorySaver, + ControllerFetcher: controllerFetcher, + RecommenderName: app.config.RecommenderName, + IgnoredNamespaces: ignoredNamespaces, + VpaObjectNamespace: app.config.CommonFlags.VpaObjectNamespace, + }.Make() + controllerFetcher.Start(ctx, scaleCacheLoopPeriod) + + recommender := routines.RecommenderFactory{ + ClusterState: clusterState, + ClusterStateFeeder: clusterStateFeeder, + ControllerFetcher: controllerFetcher, + CheckpointWriter: checkpoint.NewCheckpointWriter(clusterState, vpa_clientset.NewForConfigOrDie(config).AutoscalingV1()), + VpaClient: vpa_clientset.NewForConfigOrDie(config).AutoscalingV1(), + PodResourceRecommender: logic.CreatePodResourceRecommender(), + RecommendationPostProcessors: postProcessors, + CheckpointsGCInterval: app.config.CheckpointsGCInterval, + UseCheckpoints: useCheckpoints, + UpdateWorkerCount: app.config.UpdateWorkerCount, + }.Make() + + promQueryTimeout, err := time.ParseDuration(app.config.QueryTimeout) + if err != nil { + return fmt.Errorf("could not parse --prometheus-query-timeout as a time.Duration: %w", err) + } + + if useCheckpoints { + recommender.GetClusterStateFeeder().InitFromCheckpoints(ctx) + } else { + historyConfig := history.PrometheusHistoryProviderConfig{ + Address: app.config.PrometheusAddress, + Insecure: app.config.PrometheusInsecure, + QueryTimeout: promQueryTimeout, + HistoryLength: app.config.HistoryLength, + HistoryResolution: app.config.HistoryResolution, + PodLabelPrefix: app.config.PodLabelPrefix, + PodLabelsMetricName: app.config.PodLabelsMetricName, + PodNamespaceLabel: app.config.PodNamespaceLabel, + PodNameLabel: app.config.PodNameLabel, + CtrNamespaceLabel: app.config.CtrNamespaceLabel, + CtrPodNameLabel: app.config.CtrPodNameLabel, + CtrNameLabel: app.config.CtrNameLabel, + CadvisorMetricsJobName: app.config.PrometheusJobName, + Namespace: app.config.CommonFlags.VpaObjectNamespace, + Authentication: history.PrometheusCredentials{ + BearerToken: app.config.PrometheusBearerToken, + Username: app.config.Username, + Password: app.config.Password, + }, + } + provider, err := history.NewPrometheusHistoryProvider(historyConfig) + if err != nil { + return fmt.Errorf("could not initialize history provider: %w", err) + } + recommender.GetClusterStateFeeder().InitFromHistoryProvider(provider) + } + + // Start updating health check endpoint. + healthCheck.StartMonitoring() + + ticker := time.NewTicker(app.config.MetricsFetcherInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + recommender.RunOnce() + healthCheck.UpdateLastActivity() + } + } +} + +func (app *RecommenderApp) initGlobalMaxAllowed() apiv1.ResourceList { + result := make(apiv1.ResourceList) + if !app.config.MaxAllowedCPU.IsZero() { + result[apiv1.ResourceCPU] = app.config.MaxAllowedCPU.Quantity + } + if !app.config.MaxAllowedMemory.IsZero() { + result[apiv1.ResourceMemory] = app.config.MaxAllowedMemory.Quantity + } + return result +} + +const ( + defaultLeaseDuration = 15 * time.Second + defaultRenewDeadline = 10 * time.Second + defaultRetryPeriod = 2 * time.Second +) + +// DefaultLeaderElectionConfiguration returns the default leader election configuration +func DefaultLeaderElectionConfiguration() componentbaseconfig.LeaderElectionConfiguration { + return componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: defaultLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: defaultRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: defaultRetryPeriod}, + ResourceLock: resourcelock.LeasesResourceLock, + // This was changed from "vpa-recommender" to avoid conflicts with managed VPA deployments. + ResourceName: "vpa-recommender-lease", + ResourceNamespace: metav1.NamespaceSystem, + } +} diff --git a/vertical-pod-autoscaler/pkg/recommender/config/config.go b/vertical-pod-autoscaler/pkg/recommender/config/config.go new file mode 100644 index 000000000000..1a4dba3e5212 --- /dev/null +++ b/vertical-pod-autoscaler/pkg/recommender/config/config.go @@ -0,0 +1,212 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "flag" + "os" + "strings" + "time" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog/v2" + + "k8s.io/autoscaler/vertical-pod-autoscaler/common" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/routines" +) + +// RecommenderConfig holds all configuration for the recommender component +type RecommenderConfig struct { + // Common flags + CommonFlags *common.CommonFlags + + // Recommender-specific flags + RecommenderName string + MetricsFetcherInterval time.Duration + CheckpointsGCInterval time.Duration + Address string + Storage string + MemorySaver bool + UpdateWorkerCount int + + // Prometheus history provider configuration + PrometheusAddress string + PrometheusInsecure bool + PrometheusJobName string + HistoryLength string + HistoryResolution string + QueryTimeout string + PodLabelPrefix string + PodLabelsMetricName string + PodNamespaceLabel string + PodNameLabel string + CtrNamespaceLabel string + CtrPodNameLabel string + CtrNameLabel string + Username string + Password string + PrometheusBearerToken string + PrometheusBearerTokenFile string + + // External metrics provider configuration + UseExternalMetrics bool + ExternalCpuMetric string + ExternalMemoryMetric string + + // Aggregation configuration + MemoryAggregationInterval time.Duration + MemoryAggregationIntervalCount int64 + MemoryHistogramDecayHalfLife time.Duration + CpuHistogramDecayHalfLife time.Duration + OOMBumpUpRatio float64 + OOMMinBumpUp float64 + + // Post processors configuration + PostProcessorCPUasInteger bool + MaxAllowedCPU resource.QuantityValue + MaxAllowedMemory resource.QuantityValue +} + +// DefaultRecommenderConfig returns a RecommenderConfig with default values +func DefaultRecommenderConfig() *RecommenderConfig { + return &RecommenderConfig{ + CommonFlags: common.DefaultCommonConfig(), + + // Recommender-specific flags + RecommenderName: input.DefaultRecommenderName, + MetricsFetcherInterval: 1 * time.Minute, + CheckpointsGCInterval: 10 * time.Minute, + Address: ":8942", + Storage: "", + MemorySaver: false, + UpdateWorkerCount: 10, + + // Prometheus history provider flags + PrometheusAddress: "http://prometheus.monitoring.svc", + PrometheusInsecure: false, + PrometheusJobName: "kubernetes-cadvisor", + HistoryLength: "8d", + HistoryResolution: "1h", + QueryTimeout: "5m", + PodLabelPrefix: "pod_label_", + PodLabelsMetricName: "up{job=\"kubernetes-pods\"}", + PodNamespaceLabel: "kubernetes_namespace", + PodNameLabel: "kubernetes_pod_name", + CtrNamespaceLabel: "namespace", + CtrPodNameLabel: "pod_name", + CtrNameLabel: "name", + Username: "", + Password: "", + PrometheusBearerToken: "", + PrometheusBearerTokenFile: "", + + // External metrics provider flags + UseExternalMetrics: false, + ExternalCpuMetric: "", + ExternalMemoryMetric: "", + + // Aggregation configuration flags + MemoryAggregationInterval: model.DefaultMemoryAggregationInterval, + MemoryAggregationIntervalCount: model.DefaultMemoryAggregationIntervalCount, + MemoryHistogramDecayHalfLife: model.DefaultMemoryHistogramDecayHalfLife, + CpuHistogramDecayHalfLife: model.DefaultCPUHistogramDecayHalfLife, + OOMBumpUpRatio: model.DefaultOOMBumpUpRatio, + OOMMinBumpUp: model.DefaultOOMMinBumpUp, + + // Post processors flags + PostProcessorCPUasInteger: false, + MaxAllowedCPU: resource.QuantityValue{}, + MaxAllowedMemory: resource.QuantityValue{}, + } +} + +// InitRecommenderFlags initializes flags for the recommender component +func InitRecommenderFlags() *RecommenderConfig { + config := DefaultRecommenderConfig() + config.CommonFlags = common.InitCommonFlags() + + flag.StringVar(&config.RecommenderName, "recommender-name", config.RecommenderName, "Set the recommender name. Recommender will generate recommendations for VPAs that configure the same recommender name. If the recommender name is left as default it will also generate recommendations that don't explicitly specify recommender. You shouldn't run two recommenders with the same name in a cluster.") + flag.DurationVar(&config.MetricsFetcherInterval, "recommender-interval", config.MetricsFetcherInterval, `How often metrics should be fetched`) + flag.DurationVar(&config.CheckpointsGCInterval, "checkpoints-gc-interval", config.CheckpointsGCInterval, `How often orphaned checkpoints should be garbage collected`) + flag.StringVar(&config.Address, "address", ":8942", "The address to expose Prometheus metrics.") + flag.StringVar(&config.Storage, "storage", config.Storage, `Specifies storage mode. Supported values: prometheus, checkpoint (default)`) + flag.BoolVar(&config.MemorySaver, "memory-saver", false, `If true, only track pods which have an associated VPA`) + flag.IntVar(&config.UpdateWorkerCount, "update-worker-count", 10, "Number of concurrent workers to update VPA recommendations and checkpoints. When increasing this setting, make sure the client-side rate limits ('kube-api-qps' and 'kube-api-burst') are either increased or turned off as well. Determines the minimum number of VPA checkpoints written per recommender loop.") + + // Prometheus history provider flags + flag.StringVar(&config.PrometheusAddress, "prometheus-address", config.PrometheusAddress, `Where to reach for Prometheus metrics`) + flag.BoolVar(&config.PrometheusInsecure, "prometheus-insecure", config.PrometheusInsecure, `Skip tls verify if https is used in the prometheus-address`) + flag.StringVar(&config.PrometheusJobName, "prometheus-cadvisor-job-name", config.PrometheusJobName, `Name of the prometheus job name which scrapes the cAdvisor metrics`) + flag.StringVar(&config.HistoryLength, "history-length", config.HistoryLength, `How much time back prometheus have to be queried to get historical metrics`) + flag.StringVar(&config.HistoryResolution, "history-resolution", config.HistoryResolution, `Resolution at which Prometheus is queried for historical metrics`) + flag.StringVar(&config.QueryTimeout, "prometheus-query-timeout", config.QueryTimeout, `How long to wait before killing long queries`) + flag.StringVar(&config.PodLabelPrefix, "pod-label-prefix", config.PodLabelPrefix, `Which prefix to look for pod labels in metrics`) + flag.StringVar(&config.PodLabelsMetricName, "metric-for-pod-labels", config.PodLabelsMetricName, `Which metric to look for pod labels in metrics`) + flag.StringVar(&config.PodNamespaceLabel, "pod-namespace-label", config.PodNamespaceLabel, `Label name to look for pod namespaces`) + flag.StringVar(&config.PodNameLabel, "pod-name-label", config.PodNameLabel, `Label name to look for pod names`) + flag.StringVar(&config.CtrNamespaceLabel, "container-namespace-label", config.CtrNamespaceLabel, `Label name to look for container namespaces`) + flag.StringVar(&config.CtrPodNameLabel, "container-pod-name-label", config.CtrPodNameLabel, `Label name to look for container pod names`) + flag.StringVar(&config.CtrNameLabel, "container-name-label", config.CtrNameLabel, `Label name to look for container names`) + flag.StringVar(&config.Username, "username", config.Username, "The username used in the prometheus server basic auth. Can also be set via the PROMETHEUS_USERNAME environment variable") + flag.StringVar(&config.Password, "password", config.Password, "The password used in the prometheus server basic auth. Can also be set via the PROMETHEUS_PASSWORD environment variable") + flag.StringVar(&config.PrometheusBearerToken, "prometheus-bearer-token", config.PrometheusBearerToken, "The bearer token used in the Prometheus server bearer token auth") + flag.StringVar(&config.PrometheusBearerTokenFile, "prometheus-bearer-token-file", config.PrometheusBearerTokenFile, "Path to the bearer token file used for authentication by the Prometheus server") + + // External metrics provider flags + flag.BoolVar(&config.UseExternalMetrics, "use-external-metrics", config.UseExternalMetrics, "ALPHA. Use an external metrics provider instead of metrics_server.") + flag.StringVar(&config.ExternalCpuMetric, "external-metrics-cpu-metric", config.ExternalCpuMetric, "ALPHA. Metric to use with external metrics provider for CPU usage.") + flag.StringVar(&config.ExternalMemoryMetric, "external-metrics-memory-metric", config.ExternalMemoryMetric, "ALPHA. Metric to use with external metrics provider for memory usage.") + + // Aggregation configuration flags + flag.DurationVar(&config.MemoryAggregationInterval, "memory-aggregation-interval", config.MemoryAggregationInterval, `The length of a single interval, for which the peak memory usage is computed. Memory usage peaks are aggregated in multiples of this interval. In other words there is one memory usage sample per interval (the maximum usage over that interval)`) + flag.Int64Var(&config.MemoryAggregationIntervalCount, "memory-aggregation-interval-count", config.MemoryAggregationIntervalCount, `The number of consecutive memory-aggregation-intervals which make up the MemoryAggregationWindowLength which in turn is the period for memory usage aggregation by VPA. In other words, MemoryAggregationWindowLength = memory-aggregation-interval * memory-aggregation-interval-count.`) + flag.DurationVar(&config.MemoryHistogramDecayHalfLife, "memory-histogram-decay-half-life", config.MemoryHistogramDecayHalfLife, `The amount of time it takes a historical memory usage sample to lose half of its weight. In other words, a fresh usage sample is twice as 'important' as one with age equal to the half life period.`) + flag.DurationVar(&config.CpuHistogramDecayHalfLife, "cpu-histogram-decay-half-life", config.CpuHistogramDecayHalfLife, `The amount of time it takes a historical CPU usage sample to lose half of its weight.`) + flag.Float64Var(&config.OOMBumpUpRatio, "oom-bump-up-ratio", config.OOMBumpUpRatio, `Default memory bump up ratio when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 1.2.`) + flag.Float64Var(&config.OOMMinBumpUp, "oom-min-bump-up-bytes", config.OOMMinBumpUp, `Default minimal increase of memory (in bytes) when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 100 * 1024 * 1024 (100Mi).`) + + // Post processors flags + // CPU as integer to benefit for CPU management Static Policy ( https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy ) + flag.BoolVar(&config.PostProcessorCPUasInteger, "cpu-integer-post-processor-enabled", config.PostProcessorCPUasInteger, "Enable the cpu-integer recommendation post processor. The post processor will round up CPU recommendations to a whole CPU for pods which were opted in by setting an appropriate label on VPA object (experimental)") + flag.Var(&config.MaxAllowedCPU, "container-recommendation-max-allowed-cpu", "Maximum amount of CPU that will be recommended for a container. VerticalPodAutoscaler-level maximum allowed takes precedence over the global maximum allowed.") + flag.Var(&config.MaxAllowedMemory, "container-recommendation-max-allowed-memory", "Maximum amount of memory that will be recommended for a container. VerticalPodAutoscaler-level maximum allowed takes precedence over the global maximum allowed.") + + return config +} + +// ValidateRecommenderConfig performs validation of the recommender flags +func ValidateRecommenderConfig(config *RecommenderConfig) { + if *routines.MinCheckpointsPerRun != 10 { // Default value is 10 + klog.InfoS("DEPRECATION WARNING: The 'min-checkpoints' flag is deprecated and has no effect. It will be removed in a future release.") + } + + if config.PrometheusBearerToken != "" && config.PrometheusBearerTokenFile != "" && config.Username != "" { + klog.ErrorS(nil, "--bearer-token, --bearer-token-file and --username are mutually exclusive and can't be set together.") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } + + if config.PrometheusBearerTokenFile != "" { + fileContent, err := os.ReadFile(config.PrometheusBearerTokenFile) + if err != nil { + klog.ErrorS(err, "Unable to read bearer token file", "filename", config.PrometheusBearerTokenFile) + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } + config.PrometheusBearerTokenFile = strings.TrimSpace(string(fileContent)) + } +} diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go index ac29c207b32c..25fcbae88dfb 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go @@ -138,7 +138,12 @@ func WatchEvictionEventsWithRetries(ctx context.Context, kubeClient kube_client. // Wait between attempts, retrying too often breaks API server. waitTime := wait.Jitter(evictionWatchRetryWait, evictionWatchJitterFactor) klog.V(1).InfoS("An attempt to watch eviction events finished", "waitTime", waitTime) - time.Sleep(waitTime) + // Use a timer that can be interrupted by context cancellation + select { + case <-ctx.Done(): + return + case <-time.After(waitTime): + } } } }() diff --git a/vertical-pod-autoscaler/pkg/recommender/main.go b/vertical-pod-autoscaler/pkg/recommender/main.go index cda2d831da8c..0434d0bcf414 100644 --- a/vertical-pod-autoscaler/pkg/recommender/main.go +++ b/vertical-pod-autoscaler/pkg/recommender/main.go @@ -18,365 +18,42 @@ package main import ( "context" - "flag" - "fmt" - "os" - "strings" - "time" "github.com/spf13/pflag" - apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/informers" - kube_client "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/leaderelection" - "k8s.io/client-go/tools/leaderelection/resourcelock" kube_flag "k8s.io/component-base/cli/flag" - componentbaseconfig "k8s.io/component-base/config" componentbaseoptions "k8s.io/component-base/config/options" "k8s.io/klog/v2" - resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" "k8s.io/autoscaler/vertical-pod-autoscaler/common" - vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/checkpoint" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" - input_metrics "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/metrics" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/routines" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target" - controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" - metrics_quality "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/quality" - metrics_recommender "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/recommender" - metrics_resources "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/resources" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" - vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/app" + recommender_config "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/config" ) -var ( - recommenderName = flag.String("recommender-name", input.DefaultRecommenderName, "Set the recommender name. Recommender will generate recommendations for VPAs that configure the same recommender name. If the recommender name is left as default it will also generate recommendations that don't explicitly specify recommender. You shouldn't run two recommenders with the same name in a cluster.") - metricsFetcherInterval = flag.Duration("recommender-interval", 1*time.Minute, `How often metrics should be fetched`) - checkpointsGCInterval = flag.Duration("checkpoints-gc-interval", 10*time.Minute, `How often orphaned checkpoints should be garbage collected`) - address = flag.String("address", ":8942", "The address to expose Prometheus metrics.") - storage = flag.String("storage", "", `Specifies storage mode. Supported values: prometheus, checkpoint (default)`) - memorySaver = flag.Bool("memory-saver", false, `If true, only track pods which have an associated VPA`) - updateWorkerCount = flag.Int("update-worker-count", 10, "Number of concurrent workers to update VPA recommendations and checkpoints. When increasing this setting, make sure the client-side rate limits ('kube-api-qps' and 'kube-api-burst') are either increased or turned off as well. Determines the minimum number of VPA checkpoints written per recommender loop.") -) - -// Prometheus history provider flags -var ( - prometheusAddress = flag.String("prometheus-address", "http://prometheus.monitoring.svc", `Where to reach for Prometheus metrics`) - prometheusInsecure = flag.Bool("prometheus-insecure", false, `Skip tls verify if https is used in the prometheus-address`) - prometheusJobName = flag.String("prometheus-cadvisor-job-name", "kubernetes-cadvisor", `Name of the prometheus job name which scrapes the cAdvisor metrics`) - historyLength = flag.String("history-length", "8d", `How much time back prometheus have to be queried to get historical metrics`) - historyResolution = flag.String("history-resolution", "1h", `Resolution at which Prometheus is queried for historical metrics`) - queryTimeout = flag.String("prometheus-query-timeout", "5m", `How long to wait before killing long queries`) - podLabelPrefix = flag.String("pod-label-prefix", "pod_label_", `Which prefix to look for pod labels in metrics`) - podLabelsMetricName = flag.String("metric-for-pod-labels", "up{job=\"kubernetes-pods\"}", `Which metric to look for pod labels in metrics`) - podNamespaceLabel = flag.String("pod-namespace-label", "kubernetes_namespace", `Label name to look for pod namespaces`) - podNameLabel = flag.String("pod-name-label", "kubernetes_pod_name", `Label name to look for pod names`) - ctrNamespaceLabel = flag.String("container-namespace-label", "namespace", `Label name to look for container namespaces`) - ctrPodNameLabel = flag.String("container-pod-name-label", "pod_name", `Label name to look for container pod names`) - ctrNameLabel = flag.String("container-name-label", "name", `Label name to look for container names`) - username = flag.String("username", "", "The username used in the prometheus server basic auth. Can also be set via the PROMETHEUS_USERNAME environment variable") - password = flag.String("password", "", "The password used in the prometheus server basic auth. Can also be set via the PROMETHEUS_PASSWORD environment variable") - prometheusBearerToken = flag.String("prometheus-bearer-token", "", "The bearer token used in the Prometheus server bearer token auth") - prometheusBearerTokenFile = flag.String("prometheus-bearer-token-file", "", "Path to the bearer token file used for authentication by the Prometheus server") -) - -// External metrics provider flags -var ( - useExternalMetrics = flag.Bool("use-external-metrics", false, "ALPHA. Use an external metrics provider instead of metrics_server.") - externalCpuMetric = flag.String("external-metrics-cpu-metric", "", "ALPHA. Metric to use with external metrics provider for CPU usage.") - externalMemoryMetric = flag.String("external-metrics-memory-metric", "", "ALPHA. Metric to use with external metrics provider for memory usage.") -) - -// Aggregation configuration flags -var ( - memoryAggregationInterval = flag.Duration("memory-aggregation-interval", model.DefaultMemoryAggregationInterval, `The length of a single interval, for which the peak memory usage is computed. Memory usage peaks are aggregated in multiples of this interval. In other words there is one memory usage sample per interval (the maximum usage over that interval)`) - memoryAggregationIntervalCount = flag.Int64("memory-aggregation-interval-count", model.DefaultMemoryAggregationIntervalCount, `The number of consecutive memory-aggregation-intervals which make up the MemoryAggregationWindowLength which in turn is the period for memory usage aggregation by VPA. In other words, MemoryAggregationWindowLength = memory-aggregation-interval * memory-aggregation-interval-count.`) - memoryHistogramDecayHalfLife = flag.Duration("memory-histogram-decay-half-life", model.DefaultMemoryHistogramDecayHalfLife, `The amount of time it takes a historical memory usage sample to lose half of its weight. In other words, a fresh usage sample is twice as 'important' as one with age equal to the half life period.`) - cpuHistogramDecayHalfLife = flag.Duration("cpu-histogram-decay-half-life", model.DefaultCPUHistogramDecayHalfLife, `The amount of time it takes a historical CPU usage sample to lose half of its weight.`) - oomBumpUpRatio = flag.Float64("oom-bump-up-ratio", model.DefaultOOMBumpUpRatio, `Default memory bump up ratio when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 1.2.`) - oomMinBumpUp = flag.Float64("oom-min-bump-up-bytes", model.DefaultOOMMinBumpUp, `Default minimal increase of memory (in bytes) when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 100 * 1024 * 1024 (100Mi).`) -) - -// Post processors flags -var ( - // CPU as integer to benefit for CPU management Static Policy ( https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy ) - postProcessorCPUasInteger = flag.Bool("cpu-integer-post-processor-enabled", false, "Enable the cpu-integer recommendation post processor. The post processor will round up CPU recommendations to a whole CPU for pods which were opted in by setting an appropriate label on VPA object (experimental)") - maxAllowedCPU = resource.QuantityValue{} - maxAllowedMemory = resource.QuantityValue{} -) - -const ( - // aggregateContainerStateGCInterval defines how often expired AggregateContainerStates are garbage collected. - aggregateContainerStateGCInterval = 1 * time.Hour - scaleCacheEntryLifetime time.Duration = time.Hour - scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute - scaleCacheEntryJitterFactor float64 = 1. - scaleCacheLoopPeriod = 7 * time.Second - defaultResyncPeriod time.Duration = 10 * time.Minute -) - -func init() { - flag.Var(&maxAllowedCPU, "container-recommendation-max-allowed-cpu", "Maximum amount of CPU that will be recommended for a container. VerticalPodAutoscaler-level maximum allowed takes precedence over the global maximum allowed.") - flag.Var(&maxAllowedMemory, "container-recommendation-max-allowed-memory", "Maximum amount of memory that will be recommended for a container. VerticalPodAutoscaler-level maximum allowed takes precedence over the global maximum allowed.") -} - func main() { - commonFlags := common.InitCommonFlags() + config := recommender_config.InitRecommenderFlags() klog.InitFlags(nil) common.InitLoggingFlags() - - leaderElection := defaultLeaderElectionConfiguration() - componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) - features.MutableFeatureGate.AddFlag(pflag.CommandLine) - kube_flag.InitFlags() - klog.V(1).InfoS("Vertical Pod Autoscaler Recommender", "version", common.VerticalPodAutoscalerVersion(), "recommenderName", *recommenderName) + recommender_config.ValidateRecommenderConfig(config) + common.ValidateCommonConfig(config.CommonFlags) - if len(commonFlags.VpaObjectNamespace) > 0 && len(commonFlags.IgnoredVpaObjectNamespaces) > 0 { - klog.ErrorS(nil, "--vpa-object-namespace and --ignored-vpa-object-namespaces are mutually exclusive and can't be set together.") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } + leaderElection := app.DefaultLeaderElectionConfiguration() + componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) - if *routines.MinCheckpointsPerRun != 10 { // Default value is 10 - klog.InfoS("DEPRECATION WARNING: The 'min-checkpoints' flag is deprecated and has no effect. It will be removed in a future release.") - } + kube_flag.InitFlags() + klog.V(1).InfoS("Vertical Pod Autoscaler Recommender", "version", common.VerticalPodAutoscalerVersion(), "recommenderName", config.RecommenderName) - if *prometheusBearerToken != "" && *prometheusBearerTokenFile != "" && *username != "" { - klog.ErrorS(nil, "--bearer-token, --bearer-token-file and --username are mutually exclusive and can't be set together.") + recommenderApp, err := app.NewRecommenderApp(config) + if err != nil { + klog.ErrorS(err, "Failed to create recommender app") klog.FlushAndExit(klog.ExitFlushTimeout, 1) } - if *prometheusBearerTokenFile != "" { - fileContent, err := os.ReadFile(*prometheusBearerTokenFile) - if err != nil { - klog.ErrorS(err, "Unable to read bearer token file", "filename", *prometheusBearerTokenFile) - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } - *prometheusBearerToken = strings.TrimSpace(string(fileContent)) - } - ctx := context.Background() - - healthCheck := metrics.NewHealthCheck(*metricsFetcherInterval * 5) - metrics_recommender.Register() - metrics_quality.Register() - metrics_resources.Register() - server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) - - if !leaderElection.LeaderElect { - run(ctx, healthCheck, commonFlags) - } else { - id, err := os.Hostname() - if err != nil { - klog.ErrorS(err, "Unable to get hostname") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } - id = id + "_" + string(uuid.NewUUID()) - - config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) - kubeClient := kube_client.NewForConfigOrDie(config) - - lock, err := resourcelock.New( - leaderElection.ResourceLock, - leaderElection.ResourceNamespace, - leaderElection.ResourceName, - kubeClient.CoreV1(), - kubeClient.CoordinationV1(), - resourcelock.ResourceLockConfig{ - Identity: id, - }, - ) - if err != nil { - klog.ErrorS(err, "Unable to create leader election lock") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } - - leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ - Lock: lock, - LeaseDuration: leaderElection.LeaseDuration.Duration, - RenewDeadline: leaderElection.RenewDeadline.Duration, - RetryPeriod: leaderElection.RetryPeriod.Duration, - ReleaseOnCancel: true, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: func(_ context.Context) { - run(ctx, healthCheck, commonFlags) - }, - OnStoppedLeading: func() { - klog.Fatal("lost master") - }, - }, - }) - } -} - -const ( - defaultLeaseDuration = 15 * time.Second - defaultRenewDeadline = 10 * time.Second - defaultRetryPeriod = 2 * time.Second -) - -func defaultLeaderElectionConfiguration() componentbaseconfig.LeaderElectionConfiguration { - return componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - LeaseDuration: metav1.Duration{Duration: defaultLeaseDuration}, - RenewDeadline: metav1.Duration{Duration: defaultRenewDeadline}, - RetryPeriod: metav1.Duration{Duration: defaultRetryPeriod}, - ResourceLock: resourcelock.LeasesResourceLock, - // This was changed from "vpa-recommender" to avoid conflicts with managed VPA deployments. - ResourceName: "vpa-recommender-lease", - ResourceNamespace: metav1.NamespaceSystem, - } -} - -func run(ctx context.Context, healthCheck *metrics.HealthCheck, commonFlag *common.CommonFlags) { - // Create a stop channel that will be used to signal shutdown - stopCh := make(chan struct{}) - defer close(stopCh) - config := common.CreateKubeConfigOrDie(commonFlag.KubeConfig, float32(commonFlag.KubeApiQps), int(commonFlag.KubeApiBurst)) - kubeClient := kube_client.NewForConfigOrDie(config) - clusterState := model.NewClusterState(aggregateContainerStateGCInterval) - factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(commonFlag.VpaObjectNamespace)) - controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) - podLister, oomObserver := input.NewPodListerAndOOMObserver(ctx, kubeClient, commonFlag.VpaObjectNamespace, stopCh) - - factory.Start(stopCh) - informerMap := factory.WaitForCacheSync(stopCh) - for kind, synced := range informerMap { - if !synced { - klog.ErrorS(nil, fmt.Sprintf("Could not sync cache for the %s informer", kind.String())) - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } - } - - model.InitializeAggregationsConfig(model.NewAggregationsConfig(*memoryAggregationInterval, *memoryAggregationIntervalCount, *memoryHistogramDecayHalfLife, *cpuHistogramDecayHalfLife, *oomBumpUpRatio, *oomMinBumpUp)) - - useCheckpoints := *storage != "prometheus" - - var postProcessors []routines.RecommendationPostProcessor - if *postProcessorCPUasInteger { - postProcessors = append(postProcessors, &routines.IntegerCPUPostProcessor{}) - } - - globalMaxAllowed := initGlobalMaxAllowed() - // CappingPostProcessor, should always come in the last position for post-processing - postProcessors = append(postProcessors, routines.NewCappingRecommendationProcessor(globalMaxAllowed)) - var source input_metrics.PodMetricsLister - if *useExternalMetrics { - resourceMetrics := map[apiv1.ResourceName]string{} - if externalCpuMetric != nil && *externalCpuMetric != "" { - resourceMetrics[apiv1.ResourceCPU] = *externalCpuMetric - } - if externalMemoryMetric != nil && *externalMemoryMetric != "" { - resourceMetrics[apiv1.ResourceMemory] = *externalMemoryMetric - } - externalClientOptions := &input_metrics.ExternalClientOptions{ResourceMetrics: resourceMetrics, ContainerNameLabel: *ctrNameLabel} - klog.V(1).InfoS("Using External Metrics", "options", externalClientOptions) - source = input_metrics.NewExternalClient(config, clusterState, *externalClientOptions) - } else { - klog.V(1).InfoS("Using Metrics Server") - source = input_metrics.NewPodMetricsesSource(resourceclient.NewForConfigOrDie(config)) - } - - ignoredNamespaces := strings.Split(commonFlag.IgnoredVpaObjectNamespaces, ",") - - clusterStateFeeder := input.ClusterStateFeederFactory{ - PodLister: podLister, - OOMObserver: oomObserver, - KubeClient: kubeClient, - MetricsClient: input_metrics.NewMetricsClient(source, commonFlag.VpaObjectNamespace, "default-metrics-client"), - VpaCheckpointClient: vpa_clientset.NewForConfigOrDie(config).AutoscalingV1(), - VpaLister: vpa_api_util.NewVpasLister(vpa_clientset.NewForConfigOrDie(config), make(chan struct{}), commonFlag.VpaObjectNamespace), - VpaCheckpointLister: vpa_api_util.NewVpaCheckpointLister(vpa_clientset.NewForConfigOrDie(config), make(chan struct{}), commonFlag.VpaObjectNamespace), - ClusterState: clusterState, - SelectorFetcher: target.NewVpaTargetSelectorFetcher(config, kubeClient, factory), - MemorySaveMode: *memorySaver, - ControllerFetcher: controllerFetcher, - RecommenderName: *recommenderName, - IgnoredNamespaces: ignoredNamespaces, - VpaObjectNamespace: commonFlag.VpaObjectNamespace, - }.Make() - controllerFetcher.Start(ctx, scaleCacheLoopPeriod) - - recommender := routines.RecommenderFactory{ - ClusterState: clusterState, - ClusterStateFeeder: clusterStateFeeder, - ControllerFetcher: controllerFetcher, - CheckpointWriter: checkpoint.NewCheckpointWriter(clusterState, vpa_clientset.NewForConfigOrDie(config).AutoscalingV1()), - VpaClient: vpa_clientset.NewForConfigOrDie(config).AutoscalingV1(), - PodResourceRecommender: logic.CreatePodResourceRecommender(), - RecommendationPostProcessors: postProcessors, - CheckpointsGCInterval: *checkpointsGCInterval, - UseCheckpoints: useCheckpoints, - UpdateWorkerCount: *updateWorkerCount, - }.Make() - - promQueryTimeout, err := time.ParseDuration(*queryTimeout) - if err != nil { - klog.ErrorS(err, "Could not parse --prometheus-query-timeout as a time.Duration") + if err := recommenderApp.Run(ctx, leaderElection); err != nil { + klog.ErrorS(err, "Error running recommender") klog.FlushAndExit(klog.ExitFlushTimeout, 1) } - - if useCheckpoints { - recommender.GetClusterStateFeeder().InitFromCheckpoints(ctx) - } else { - config := history.PrometheusHistoryProviderConfig{ - Address: *prometheusAddress, - Insecure: *prometheusInsecure, - QueryTimeout: promQueryTimeout, - HistoryLength: *historyLength, - HistoryResolution: *historyResolution, - PodLabelPrefix: *podLabelPrefix, - PodLabelsMetricName: *podLabelsMetricName, - PodNamespaceLabel: *podNamespaceLabel, - PodNameLabel: *podNameLabel, - CtrNamespaceLabel: *ctrNamespaceLabel, - CtrPodNameLabel: *ctrPodNameLabel, - CtrNameLabel: *ctrNameLabel, - CadvisorMetricsJobName: *prometheusJobName, - Namespace: commonFlag.VpaObjectNamespace, - Authentication: history.PrometheusCredentials{ - BearerToken: *prometheusBearerToken, - Username: *username, - Password: *password, - }, - } - provider, err := history.NewPrometheusHistoryProvider(config) - if err != nil { - klog.ErrorS(err, "Could not initialize history provider") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } - recommender.GetClusterStateFeeder().InitFromHistoryProvider(provider) - } - - // Start updating health check endpoint. - healthCheck.StartMonitoring() - - ticker := time.Tick(*metricsFetcherInterval) - for range ticker { - recommender.RunOnce() - healthCheck.UpdateLastActivity() - } -} - -func initGlobalMaxAllowed() apiv1.ResourceList { - result := make(apiv1.ResourceList) - if !maxAllowedCPU.IsZero() { - result[apiv1.ResourceCPU] = maxAllowedCPU.Quantity - } - if !maxAllowedMemory.IsZero() { - result[apiv1.ResourceMemory] = maxAllowedMemory.Quantity - } - - return result } diff --git a/vertical-pod-autoscaler/pkg/target/controller_fetcher/controller_fetcher.go b/vertical-pod-autoscaler/pkg/target/controller_fetcher/controller_fetcher.go index d65eeb016958..788b83e0d1b2 100644 --- a/vertical-pod-autoscaler/pkg/target/controller_fetcher/controller_fetcher.go +++ b/vertical-pod-autoscaler/pkg/target/controller_fetcher/controller_fetcher.go @@ -112,7 +112,7 @@ func (f *controllerFetcher) Start(ctx context.Context, loopPeriod time.Duration) } // NewControllerFetcher returns a new instance of controllerFetcher -func NewControllerFetcher(config *rest.Config, kubeClient kube_client.Interface, factory informers.SharedInformerFactory, betweenRefreshes, lifeTime time.Duration, jitterFactor float64) *controllerFetcher { +func NewControllerFetcher(ctx context.Context, config *rest.Config, kubeClient kube_client.Interface, factory informers.SharedInformerFactory, betweenRefreshes, lifeTime time.Duration, jitterFactor float64) *controllerFetcher { discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) if err != nil { klog.ErrorS(err, "Could not create discoveryClient") @@ -122,9 +122,9 @@ func NewControllerFetcher(config *rest.Config, kubeClient kube_client.Interface, restClient := kubeClient.CoreV1().RESTClient() cachedDiscoveryClient := cacheddiscovery.NewMemCacheClient(discoveryClient) mapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient) - go wait.Until(func() { + go wait.UntilWithContext(ctx, func(ctx context.Context) { mapper.Reset() - }, discoveryResetPeriod, make(chan struct{})) + }, discoveryResetPeriod) informersMap := map[wellKnownController]cache.SharedIndexInformer{ daemonSet: factory.Apps().V1().DaemonSets().Informer(), diff --git a/vertical-pod-autoscaler/pkg/target/fetcher.go b/vertical-pod-autoscaler/pkg/target/fetcher.go index 88789137ccd5..7f544c225763 100644 --- a/vertical-pod-autoscaler/pkg/target/fetcher.go +++ b/vertical-pod-autoscaler/pkg/target/fetcher.go @@ -68,7 +68,7 @@ const ( ) // NewVpaTargetSelectorFetcher returns new instance of VpaTargetSelectorFetcher -func NewVpaTargetSelectorFetcher(config *rest.Config, kubeClient kube_client.Interface, factory informers.SharedInformerFactory) VpaTargetSelectorFetcher { +func NewVpaTargetSelectorFetcher(ctx context.Context, config *rest.Config, kubeClient kube_client.Interface, factory informers.SharedInformerFactory) VpaTargetSelectorFetcher { discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) if err != nil { klog.ErrorS(err, "Could not create discoveryClient") @@ -78,9 +78,9 @@ func NewVpaTargetSelectorFetcher(config *rest.Config, kubeClient kube_client.Int restClient := kubeClient.CoreV1().RESTClient() cachedDiscoveryClient := cacheddiscovery.NewMemCacheClient(discoveryClient) mapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient) - go wait.Until(func() { + go wait.UntilWithContext(ctx, func(ctx context.Context) { mapper.Reset() - }, discoveryResetPeriod, make(chan struct{})) + }, discoveryResetPeriod) informersMap := map[wellKnownController]cache.SharedIndexInformer{ daemonSet: factory.Apps().V1().DaemonSets().Informer(), diff --git a/vertical-pod-autoscaler/pkg/updater/config/config.go b/vertical-pod-autoscaler/pkg/updater/config/config.go new file mode 100644 index 000000000000..4f258c1498da --- /dev/null +++ b/vertical-pod-autoscaler/pkg/updater/config/config.go @@ -0,0 +1,84 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "flag" + "os" + "time" + + "k8s.io/klog/v2" + + "k8s.io/autoscaler/vertical-pod-autoscaler/common" +) + +// UpdaterConfig holds all configuration for the admission controller component +type UpdaterConfig struct { + // Common flags + CommonFlags *common.CommonFlags + + UpdaterInterval time.Duration + MinReplicas int + EvictionToleranceFraction float64 + EvictionRateLimit float64 + EvictionRateBurst int + Namespace string + Address string + UseAdmissionControllerStatus bool + InPlaceSkipDisruptionBudget bool +} + +// DefaultUpdaterConfig returns a UpdaterConfig with default values +func DefaultUpdaterConfig() *UpdaterConfig { + return &UpdaterConfig{ + CommonFlags: common.DefaultCommonConfig(), + UpdaterInterval: 1 * time.Minute, + MinReplicas: 2, + EvictionToleranceFraction: 0.5, + EvictionRateLimit: -1, + EvictionRateBurst: 1, + Namespace: os.Getenv("NAMESPACE"), + Address: ":8943", + UseAdmissionControllerStatus: true, + InPlaceSkipDisruptionBudget: false, + } +} + +// InitUpdaterFlags initializes flags for the updater component +func InitUpdaterFlags() *UpdaterConfig { + config := DefaultUpdaterConfig() + config.CommonFlags = common.InitCommonFlags() + + flag.DurationVar(&config.UpdaterInterval, "updater-interval", config.UpdaterInterval, "How often updater should run") + flag.IntVar(&config.MinReplicas, "min-replicas", config.MinReplicas, "Minimum number of replicas to perform update") + flag.Float64Var(&config.EvictionToleranceFraction, "eviction-tolerance", config.EvictionToleranceFraction, "Fraction of replica count that can be evicted for update, if more than one pod can be evicted.") + flag.Float64Var(&config.EvictionRateLimit, "eviction-rate-limit", config.EvictionRateLimit, "Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable the rate limiter.") + flag.IntVar(&config.EvictionRateBurst, "eviction-rate-burst", config.EvictionRateBurst, "Burst of pods that can be evicted.") + flag.StringVar(&config.Address, "address", config.Address, "The address to expose Prometheus metrics.") + flag.BoolVar(&config.UseAdmissionControllerStatus, "use-admission-controller-status", config.UseAdmissionControllerStatus, "If true, updater will only evict pods when admission controller status is valid.") + flag.BoolVar(&config.InPlaceSkipDisruptionBudget, "in-place-skip-disruption-budget", config.InPlaceSkipDisruptionBudget, "[ALPHA] If true, VPA updater skips disruption budget checks for in-place pod updates when all containers have NotRequired resize policy (or no policy defined) for both CPU and memory resources. Disruption budgets are still respected when any container has RestartContainer resize policy for any resource.") + + return config +} + +// ValidateUpdaterConfig performs validation of the updater flags +func ValidateUpdaterConfig(config *UpdaterConfig) { + if len(config.CommonFlags.VpaObjectNamespace) > 0 && len(config.CommonFlags.IgnoredVpaObjectNamespaces) > 0 { + klog.ErrorS(nil, "--vpa-object-namespace and --ignored-vpa-object-namespaces are mutually exclusive and can't be set together.") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } +} diff --git a/vertical-pod-autoscaler/pkg/updater/main.go b/vertical-pod-autoscaler/pkg/updater/main.go index 409c72256517..1af1748834f6 100644 --- a/vertical-pod-autoscaler/pkg/updater/main.go +++ b/vertical-pod-autoscaler/pkg/updater/main.go @@ -18,7 +18,6 @@ package main import ( "context" - "flag" "fmt" "os" "strings" @@ -43,6 +42,7 @@ import ( "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target" controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + updater_config "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/config" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/inplace" updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/logic" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/priority" @@ -54,37 +54,6 @@ import ( vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" ) -var ( - updaterInterval = flag.Duration("updater-interval", 1*time.Minute, - `How often updater should run`) - - minReplicas = flag.Int("min-replicas", 2, - `Minimum number of replicas to perform update`) - - evictionToleranceFraction = flag.Float64("eviction-tolerance", 0.5, - `Fraction of replica count that can be evicted for update, if more than one pod can be evicted.`) - - evictionRateLimit = flag.Float64("eviction-rate-limit", -1, - `Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable - the rate limiter.`) - - evictionRateBurst = flag.Int("eviction-rate-burst", 1, `Burst of pods that can be evicted.`) - - address = flag.String("address", ":8943", "The address to expose Prometheus metrics.") - - useAdmissionControllerStatus = flag.Bool("use-admission-controller-status", true, - "If true, updater will only evict pods when admission controller status is valid.") - - inPlaceSkipDisruptionBudget = flag.Bool( - "in-place-skip-disruption-budget", - false, - "[ALPHA] If true, VPA updater skips disruption budget checks for in-place pod updates when all containers have NotRequired resize policy (or no policy defined) for both CPU and memory resources. "+ - "Disruption budgets are still respected when any container has RestartContainer resize policy for any resource.", - ) - - namespace = os.Getenv("NAMESPACE") -) - const ( defaultResyncPeriod time.Duration = 10 * time.Minute scaleCacheEntryLifetime time.Duration = time.Hour @@ -92,8 +61,10 @@ const ( scaleCacheEntryJitterFactor float64 = 1. ) +var config *updater_config.UpdaterConfig + func main() { - commonFlags := common.InitCommonFlags() + config = updater_config.InitUpdaterFlags() klog.InitFlags(nil) common.InitLoggingFlags() @@ -105,18 +76,13 @@ func main() { kube_flag.InitFlags() klog.V(1).InfoS("Vertical Pod Autoscaler Updater", "version", common.VerticalPodAutoscalerVersion()) - if len(commonFlags.VpaObjectNamespace) > 0 && len(commonFlags.IgnoredVpaObjectNamespaces) > 0 { - klog.ErrorS(nil, "--vpa-object-namespace and --ignored-vpa-object-namespaces are mutually exclusive and can't be set together.") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) - } - - healthCheck := metrics.NewHealthCheck(*updaterInterval * 5) - server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) + healthCheck := metrics.NewHealthCheck(config.UpdaterInterval * 5) + server.Initialize(&config.CommonFlags.EnableProfiling, healthCheck, &config.Address) metrics_updater.Register() if !leaderElection.LeaderElect { - run(healthCheck, commonFlags) + run(healthCheck, config.CommonFlags) } else { id, err := os.Hostname() if err != nil { @@ -125,8 +91,8 @@ func main() { } id = id + "_" + string(uuid.NewUUID()) - config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) - kubeClient := kube_client.NewForConfigOrDie(config) + kubeConfig := common.CreateKubeConfigOrDie(config.CommonFlags.KubeConfig, float32(config.CommonFlags.KubeApiQps), int(config.CommonFlags.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(kubeConfig) lock, err := resourcelock.New( leaderElection.ResourceLock, @@ -151,7 +117,7 @@ func main() { ReleaseOnCancel: true, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: func(_ context.Context) { - run(healthCheck, commonFlags) + run(healthCheck, config.CommonFlags) }, OnStoppedLeading: func() { klog.Fatal("lost master") @@ -182,12 +148,16 @@ func defaultLeaderElectionConfiguration() componentbaseconfig.LeaderElectionConf func run(healthCheck *metrics.HealthCheck, commonFlag *common.CommonFlags) { stopCh := make(chan struct{}) defer close(stopCh) - config := common.CreateKubeConfigOrDie(commonFlag.KubeConfig, float32(commonFlag.KubeApiQps), int(commonFlag.KubeApiBurst)) - kubeClient := kube_client.NewForConfigOrDie(config) - vpaClient := vpa_clientset.NewForConfigOrDie(config) + ctx := context.Background() + + kubeConfig := common.CreateKubeConfigOrDie(commonFlag.KubeConfig, float32(commonFlag.KubeApiQps), int(commonFlag.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(kubeConfig) + vpaClient := vpa_clientset.NewForConfigOrDie(kubeConfig) + factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(commonFlag.VpaObjectNamespace)) - targetSelectorFetcher := target.NewVpaTargetSelectorFetcher(config, kubeClient, factory) - controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + targetSelectorFetcher := target.NewVpaTargetSelectorFetcher(ctx, kubeConfig, kubeClient, factory) + controllerFetcher := controllerfetcher.NewControllerFetcher(ctx, kubeConfig, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + var limitRangeCalculator limitrange.LimitRangeCalculator limitRangeCalculator, err := limitrange.NewLimitsRangeCalculator(factory) if err != nil { @@ -205,8 +175,8 @@ func run(healthCheck *metrics.HealthCheck, commonFlag *common.CommonFlags) { } admissionControllerStatusNamespace := status.AdmissionControllerStatusNamespace - if namespace != "" { - admissionControllerStatusNamespace = namespace + if config.Namespace != "" { + admissionControllerStatusNamespace = config.Namespace } ignoredNamespaces := strings.Split(commonFlag.IgnoredVpaObjectNamespaces, ",") @@ -219,12 +189,12 @@ func run(healthCheck *metrics.HealthCheck, commonFlag *common.CommonFlags) { updater, err := updater.NewUpdater( kubeClient, vpaClient, - *minReplicas, - *evictionRateLimit, - *evictionRateBurst, - *evictionToleranceFraction, - *useAdmissionControllerStatus, - *inPlaceSkipDisruptionBudget, + config.MinReplicas, + config.EvictionRateLimit, + config.EvictionRateBurst, + config.EvictionToleranceFraction, + config.UseAdmissionControllerStatus, + config.InPlaceSkipDisruptionBudget, admissionControllerStatusNamespace, vpa_api_util.NewCappingRecommendationProcessor(limitRangeCalculator), priority.NewScalingDirectionPodEvictionAdmission(), @@ -243,9 +213,9 @@ func run(healthCheck *metrics.HealthCheck, commonFlag *common.CommonFlags) { // Start updating health check endpoint. healthCheck.StartMonitoring() - ticker := time.Tick(*updaterInterval) + ticker := time.Tick(config.UpdaterInterval) for range ticker { - ctx, cancel := context.WithTimeout(context.Background(), *updaterInterval) + ctx, cancel := context.WithTimeout(context.Background(), config.UpdaterInterval) updater.RunOnce(ctx) healthCheck.UpdateLastActivity() cancel() diff --git a/vertical-pod-autoscaler/pkg/utils/metrics/quality/quality.go b/vertical-pod-autoscaler/pkg/utils/metrics/quality/quality.go index 11c3d18ce96e..2cc1e6c67931 100644 --- a/vertical-pod-autoscaler/pkg/utils/metrics/quality/quality.go +++ b/vertical-pod-autoscaler/pkg/utils/metrics/quality/quality.go @@ -120,15 +120,15 @@ var ( // Register initializes all VPA quality metrics func Register() { - prometheus.MustRegister(usageRecommendationRelativeDiff) - prometheus.MustRegister(usageMissingRecommendationCounter) - prometheus.MustRegister(cpuRecommendationOverUsageDiff) - prometheus.MustRegister(memoryRecommendationOverUsageDiff) - prometheus.MustRegister(cpuRecommendationLowerOrEqualUsageDiff) - prometheus.MustRegister(memoryRecommendationLowerOrEqualUsageDiff) - prometheus.MustRegister(cpuRecommendations) - prometheus.MustRegister(memoryRecommendations) - prometheus.MustRegister(relativeRecommendationChange) + _ = prometheus.Register(usageRecommendationRelativeDiff) + _ = prometheus.Register(usageMissingRecommendationCounter) + _ = prometheus.Register(cpuRecommendationOverUsageDiff) + _ = prometheus.Register(memoryRecommendationOverUsageDiff) + _ = prometheus.Register(cpuRecommendationLowerOrEqualUsageDiff) + _ = prometheus.Register(memoryRecommendationLowerOrEqualUsageDiff) + _ = prometheus.Register(cpuRecommendations) + _ = prometheus.Register(memoryRecommendations) + _ = prometheus.Register(relativeRecommendationChange) } // observeUsageRecommendationRelativeDiff records relative diff between usage and diff --git a/vertical-pod-autoscaler/pkg/utils/metrics/recommender/recommender.go b/vertical-pod-autoscaler/pkg/utils/metrics/recommender/recommender.go index cc81e3dd954c..a6b9e0e08cc5 100644 --- a/vertical-pod-autoscaler/pkg/utils/metrics/recommender/recommender.go +++ b/vertical-pod-autoscaler/pkg/utils/metrics/recommender/recommender.go @@ -115,7 +115,19 @@ type ObjectCounter struct { // Register initializes all metrics for VPA Recommender func Register() { - prometheus.MustRegister(vpaObjectCount, recommendationLatency, functionLatency, aggregateContainerStatesCount, metricServerResponses, prometheusClientRequestsCount, prometheusClientRequestsDuration) + collectors := []prometheus.Collector{ + vpaObjectCount, + recommendationLatency, + functionLatency, + aggregateContainerStatesCount, + metricServerResponses, + prometheusClientRequestsCount, + prometheusClientRequestsDuration, + } + for _, c := range collectors { + // Ignore AlreadyRegisteredError + _ = prometheus.Register(c) + } } // NewExecutionTimer provides a timer for Recommender's RunOnce execution diff --git a/vertical-pod-autoscaler/pkg/utils/metrics/resources/resources.go b/vertical-pod-autoscaler/pkg/utils/metrics/resources/resources.go index 8c2badf5f9ac..9ab03fdbd7cf 100644 --- a/vertical-pod-autoscaler/pkg/utils/metrics/resources/resources.go +++ b/vertical-pod-autoscaler/pkg/utils/metrics/resources/resources.go @@ -56,7 +56,7 @@ var ( // Register initializes all metrics for VPA resources func Register() { - prometheus.MustRegister(getResourcesCount) + _ = prometheus.Register(getResourcesCount) } // RecordGetResourcesCount records how many times VPA requested the resources ( diff --git a/vertical-pod-autoscaler/pkg/utils/server/server.go b/vertical-pod-autoscaler/pkg/utils/server/server.go index 20b98cb8b820..a88806e6242f 100644 --- a/vertical-pod-autoscaler/pkg/utils/server/server.go +++ b/vertical-pod-autoscaler/pkg/utils/server/server.go @@ -18,6 +18,7 @@ limitations under the License. package server import ( + "context" "net/http" "net/http/pprof" @@ -29,6 +30,12 @@ import ( // Initialize sets up Prometheus to expose metrics & (optionally) health-check and profiling on the given address func Initialize(enableProfiling *bool, healthCheck *metrics.HealthCheck, address *string) { + InitializeWithContext(context.Background(), enableProfiling, healthCheck, address) +} + +// InitializeWithContext sets up Prometheus to expose metrics & (optionally) health-check and profiling on the given address. +// The server will shut down gracefully when the context is canceled. +func InitializeWithContext(ctx context.Context, enableProfiling *bool, healthCheck *metrics.HealthCheck, address *string) { go func() { mux := http.NewServeMux() @@ -45,8 +52,23 @@ func Initialize(enableProfiling *bool, healthCheck *metrics.HealthCheck, address mux.HandleFunc("/debug/pprof/trace", pprof.Trace) } - err := http.ListenAndServe(*address, mux) - klog.ErrorS(err, "Failed to start metrics") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) + server := &http.Server{ + Addr: *address, + Handler: mux, + } + + // Start server shutdown when context is canceled + go func() { + <-ctx.Done() + if err := server.Shutdown(context.Background()); err != nil { + klog.ErrorS(err, "Failed to shutdown metrics server") + } + }() + + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + klog.ErrorS(err, "Failed to start metrics") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } }() } diff --git a/vertical-pod-autoscaler/pkg/utils/test/test_container.go b/vertical-pod-autoscaler/pkg/utils/test/test_container.go index 685a9d86d1dd..5b911eeab40e 100644 --- a/vertical-pod-autoscaler/pkg/utils/test/test_container.go +++ b/vertical-pod-autoscaler/pkg/utils/test/test_container.go @@ -23,6 +23,7 @@ import ( type containerBuilder struct { name string + image string cpuRequest *resource.Quantity memRequest *resource.Quantity cpuLimit *resource.Quantity @@ -41,6 +42,12 @@ func (cb *containerBuilder) WithName(name string) *containerBuilder { return &r } +func (cb *containerBuilder) WithImage(image string) *containerBuilder { + r := *cb + r.image = image + return &r +} + func (cb *containerBuilder) WithCPURequest(cpuRequest resource.Quantity) *containerBuilder { r := *cb r.cpuRequest = &cpuRequest @@ -73,7 +80,8 @@ func (cb *containerBuilder) WithContainerResizePolicy(resizePolicy []apiv1.Conta func (cb *containerBuilder) Get() apiv1.Container { container := apiv1.Container{ - Name: cb.name, + Name: cb.name, + Image: cb.image, Resources: apiv1.ResourceRequirements{ Requests: apiv1.ResourceList{}, Limits: apiv1.ResourceList{}, diff --git a/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go b/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go index 8c3aeeb88650..c5c1c201bc91 100644 --- a/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go +++ b/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go @@ -27,6 +27,13 @@ import ( vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" ) +// HamsterTargetRef is CrossVersionObjectReference of hamster app +var HamsterTargetRef = &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster-deployment", +} + // VerticalPodAutoscalerBuilder helps building test instances of VerticalPodAutoscaler. type VerticalPodAutoscalerBuilder interface { WithName(vpaName string) VerticalPodAutoscalerBuilder