diff --git a/docs/kubernetes/index.md b/docs/kubernetes/index.md index 567a0c646..ed700fc40 100644 --- a/docs/kubernetes/index.md +++ b/docs/kubernetes/index.md @@ -378,12 +378,20 @@ spec: poolMin: 5 ``` -Optional: add `scaleStrategy` to limit the pace of scaling: +The pool buffer counts only unallocated pods that are Ready. Pods that are still +starting count toward the pool's total capacity, but are not advertised as +available buffer. + +Optional: add `scaleStrategy` to limit the size of each scale-up and scale-down +batch (the default is `25%`): ```yaml scaleStrategy: maxUnavailable: "20%" # or absolute number like 5 ``` +The controller waits for a scale-down batch to finish terminating before it +starts another scaling batch. + Create a batch of sandboxes using the pool: ```yaml diff --git a/kubernetes/charts/opensandbox-controller/templates/crds/pools.yaml b/kubernetes/charts/opensandbox-controller/templates/crds/pools.yaml index 148d137f7..71a31f194 100644 --- a/kubernetes/charts/opensandbox-controller/templates/crds/pools.yaml +++ b/kubernetes/charts/opensandbox-controller/templates/crds/pools.yaml @@ -36,6 +36,10 @@ spec: jsonPath: .status.available name: AVAILABLE type: integer + - description: The number of nodes updated to the latest revision. + jsonPath: .status.updated + name: UPDATED + type: integer name: v1alpha1 schema: openAPIV3Schema: @@ -169,6 +173,11 @@ spec: description: Total is the total number of nodes in the pool. format: int32 type: integer + updated: + description: Updated is the number of nodes that have been updated + to the latest revision. + format: int32 + type: integer required: - allocated - available diff --git a/kubernetes/internal/controller/pool_controller.go b/kubernetes/internal/controller/pool_controller.go index d23534385..caea70824 100644 --- a/kubernetes/internal/controller/pool_controller.go +++ b/kubernetes/internal/controller/pool_controller.go @@ -152,14 +152,18 @@ func (r *PoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul log.Error(err, "Failed to list pods") return reconcile.Result{}, err } + controllerKey := controllerutils.GetControllerKey(pool) + podNames := make(map[string]struct{}, len(podList.Items)) pods := make([]*corev1.Pod, 0, len(podList.Items)) for i := range podList.Items { pod := podList.Items[i] - PoolScaleExpectations.ObserveScale(controllerutils.GetControllerKey(pool), expectations.Create, pod.Name) + podNames[pod.Name] = struct{}{} + PoolScaleExpectations.ObserveScale(controllerKey, expectations.Create, pod.Name) if pod.DeletionTimestamp.IsZero() { pods = append(pods, &pod) } } + observeDeletedPods(controllerKey, podNames) // List all batch sandboxes ref to the pool batchSandboxList := &sandboxv1alpha1.BatchSandboxList{} @@ -179,11 +183,19 @@ func (r *PoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul batchSandboxes = append(batchSandboxes, &batchSandbox) } log.Info("Pool reconcile", "pool", pool.Name, "pods", len(pods), "batchSandboxes", len(batchSandboxes)) - return r.reconcilePool(ctx, pool, batchSandboxes, pods) + return r.reconcilePool(ctx, pool, batchSandboxes, pods, int32(len(podList.Items))) +} + +func observeDeletedPods(controllerKey string, existingPodNames map[string]struct{}) { + for name := range PoolScaleExpectations.GetExpectations(controllerKey)[expectations.Delete] { + if _, exists := existingPodNames[name]; !exists { + PoolScaleExpectations.ObserveScale(controllerKey, expectations.Delete, name) + } + } } // reconcilePool contains the main reconciliation logic -func (r *PoolReconciler) reconcilePool(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod) (ctrl.Result, error) { +func (r *PoolReconciler) reconcilePool(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod, totalPodCnt int32) (ctrl.Result, error) { var result ctrl.Result err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { @@ -220,16 +232,20 @@ func (r *PoolReconciler) reconcilePool(ctx context.Context, pool *sandboxv1alpha args := &scaleArgs{ updateRevision: updateResult.UpdateRevision, pods: schedulePods, - totalPodCnt: int32(len(pods)), + totalPodCnt: totalPodCnt, allocatedCnt: int32(len(schedResult.LatestAllocation)), idlePods: updateResult.IdlePods, toDeletePods: toDeletePods, supplyCnt: schedResult.SupplyCnt + updateResult.SupplyUpdateRevision, } - if err := r.scalePool(ctx, latestPool, args); err != nil { + scalePending, err := r.scalePool(ctx, latestPool, args) + if err != nil { return err } + if scalePending { + result = ctrl.Result{RequeueAfter: defaultRetryTime} + } // 6. Update pool status if err := r.updatePoolStatus(ctx, updateResult.UpdateRevision, latestPool, pods, schedulePods, schedResult.LatestAllocation); err != nil { @@ -709,7 +725,7 @@ type UpdateResult struct { SupplyUpdateRevision int32 } -func (r *PoolReconciler) scalePool(ctx context.Context, pool *sandboxv1alpha1.Pool, args *scaleArgs) error { +func (r *PoolReconciler) scalePool(ctx context.Context, pool *sandboxv1alpha1.Pool, args *scaleArgs) (bool, error) { log := logf.FromContext(ctx) errs := make([]error, 0) pods := args.pods @@ -720,7 +736,7 @@ func (r *PoolReconciler) scalePool(ctx context.Context, pool *sandboxv1alpha1.Po PoolScaleExpectations.DeleteExpectations(controllerutils.GetControllerKey(pool)) } else { log.Info("Pool scale is not ready, requeue", "unsatisfiedDuration", unsatisfiedDuration, "dirtyPods", dirtyPods) - return fmt.Errorf("pool scale is not ready, %v", pool.Name) + return true, nil } } schedulableCnt := int32(len(args.pods)) @@ -728,13 +744,10 @@ func (r *PoolReconciler) scalePool(ctx context.Context, pool *sandboxv1alpha1.Po allocatedCnt := args.allocatedCnt supplyCnt := args.supplyCnt toDeletePods := args.toDeletePods - bufferCnt := schedulableCnt - allocatedCnt + bufferCnt := r.countReadyIdlePods(pods, args.idlePods) // Calculate desired buffer cnt. - desiredBufferCnt := bufferCnt - if bufferCnt < pool.Spec.CapacitySpec.BufferMin || bufferCnt > pool.Spec.CapacitySpec.BufferMax { - desiredBufferCnt = (pool.Spec.CapacitySpec.BufferMin + pool.Spec.CapacitySpec.BufferMax) / 2 - } + desiredBufferCnt := desiredBufferCount(bufferCnt, pool.Spec.CapacitySpec.BufferMin, pool.Spec.CapacitySpec.BufferMax) // Calculate desired schedulable cnt. desiredSchedulableCnt := max(allocatedCnt+supplyCnt+desiredBufferCnt, pool.Spec.CapacitySpec.PoolMin) @@ -775,10 +788,20 @@ func (r *PoolReconciler) scalePool(ctx context.Context, pool *sandboxv1alpha1.Po } if scaleIn > 0 || len(toDeletePods) > 0 { podsToDelete := r.pickPodsToDelete(pods, args.idlePods, args.toDeletePods, scaleIn) + maxDeleteCnt := r.getScaleMaxUnavailable(pool, desiredSchedulableCnt) + if int32(len(podsToDelete)) > maxDeleteCnt { + podsToDelete = podsToDelete[:maxDeleteCnt] + } log.Info("Scaling down pool", "pool", pool.Name, "scaleIn", scaleIn, "toDeletePods", len(toDeletePods), "podsToDelete", len(podsToDelete)) for _, pod := range podsToDelete { log.Info("Deleting pool pod", "pool", pool.Name, "pod", pod.Name) + controllerKey := controllerutils.GetControllerKey(pool) + PoolScaleExpectations.ExpectScale(controllerKey, expectations.Delete, pod.Name) if err := r.Delete(ctx, pod); err != nil { + PoolScaleExpectations.ObserveScale(controllerKey, expectations.Delete, pod.Name) + if errors.IsNotFound(err) { + continue + } log.Error(err, "Failed to delete pool pod", "pod", pod.Name) r.Recorder.Eventf(pool, corev1.EventTypeWarning, EventReasonFailedDelete, "Failed to delete pool pod %s: %v", pod.Name, err) errs = append(errs, err) @@ -787,7 +810,7 @@ func (r *PoolReconciler) scalePool(ctx context.Context, pool *sandboxv1alpha1.Po } } } - return gerrors.Join(errs...) + return false, gerrors.Join(errs...) } func (r *PoolReconciler) updatePoolStatus(ctx context.Context, updateRevision string, pool *sandboxv1alpha1.Pool, pods []*corev1.Pod, schedulePods []*corev1.Pod, podAllocation map[string]string) error { @@ -832,38 +855,78 @@ func (r *PoolReconciler) pickPodsToDelete(pods []*corev1.Pod, idlePodNames []str podMap[pod.Name] = pod } + selected := make(map[string]struct{}) var podsToDelete []*corev1.Pod for _, name := range toDeletePodNames { pod, ok := podMap[name] - if !ok { + if !ok || !pod.DeletionTimestamp.IsZero() { + continue + } + if _, exists := selected[name]; exists { continue } podsToDelete = append(podsToDelete, pod) + selected[name] = struct{}{} } var idlePods []*corev1.Pod for _, name := range idlePodNames { pod, ok := podMap[name] - if !ok { + if !ok || !pod.DeletionTimestamp.IsZero() { continue } idlePods = append(idlePods, pod) } sort.Slice(idlePods, func(i, j int) bool { - return idlePods[i].CreationTimestamp.Before(&idlePods[j].CreationTimestamp) + iReady := utils.IsPodReady(idlePods[i]) + jReady := utils.IsPodReady(idlePods[j]) + if iReady != jReady { + return !iReady + } + if idlePods[i].CreationTimestamp.Equal(&idlePods[j].CreationTimestamp) { + return idlePods[i].Name < idlePods[j].Name + } + return idlePods[i].CreationTimestamp.After(idlePods[j].CreationTimestamp.Time) }) for _, pod := range idlePods { if scaleIn <= 0 { break } - if pod.DeletionTimestamp == nil { - podsToDelete = append(podsToDelete, pod) + if _, exists := selected[pod.Name]; exists { + continue } + podsToDelete = append(podsToDelete, pod) + selected[pod.Name] = struct{}{} scaleIn -= 1 } return podsToDelete } +func desiredBufferCount(readyBufferCnt, bufferMin, bufferMax int32) int32 { + if readyBufferCnt < bufferMin { + return bufferMin + } + if readyBufferCnt > bufferMax { + return bufferMax + } + return readyBufferCnt +} + +func (r *PoolReconciler) countReadyIdlePods(pods []*corev1.Pod, idlePodNames []string) int32 { + idleNames := make(map[string]struct{}, len(idlePodNames)) + for _, name := range idlePodNames { + idleNames[name] = struct{}{} + } + + var count int32 + for _, pod := range pods { + if _, idle := idleNames[pod.Name]; idle && utils.IsPodReady(pod) { + count++ + } + } + return count +} + // getScaleMaxUnavailable returns the resolved maxUnavailable value. // If not specified, defaults to 25% of desiredTotal. // Minimum return value is 1 to ensure scaling progress. diff --git a/kubernetes/internal/controller/pool_scaling_stability_test.go b/kubernetes/internal/controller/pool_scaling_stability_test.go new file mode 100644 index 000000000..b51161e68 --- /dev/null +++ b/kubernetes/internal/controller/pool_scaling_stability_test.go @@ -0,0 +1,358 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/controller/algorithm" + controllerutils "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/controller" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/expectations" +) + +func stabilityTestPod(name string, ready bool, created time.Time) *corev1.Pod { + status := corev1.ConditionFalse + phase := corev1.PodPending + if ready { + status = corev1.ConditionTrue + phase = corev1.PodRunning + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + CreationTimestamp: metav1.NewTime(created), + }, + Status: corev1.PodStatus{Phase: phase, Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: status, + }}}, + } +} + +func stabilityTestPool(maxUnavailable intstr.IntOrString) *sandboxv1alpha1.Pool { + return &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{Name: "pool", Namespace: "default", UID: "pool-uid"}, + Spec: sandboxv1alpha1.PoolSpec{ + CapacitySpec: sandboxv1alpha1.CapacitySpec{ + PoolMin: 0, PoolMax: 100, BufferMin: 0, BufferMax: 0, + }, + ScaleStrategy: &sandboxv1alpha1.ScaleStrategy{MaxUnavailable: &maxUnavailable}, + }, + } +} + +func stabilityTestReconciler(t *testing.T, objects ...client.Object) *PoolReconciler { + t.Helper() + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := sandboxv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + return &PoolReconciler{ + Client: fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(&sandboxv1alpha1.Pool{}). + Build(), + Scheme: scheme, + Recorder: record.NewFakeRecorder(20), + } +} + +func TestCountReadyIdlePods(t *testing.T) { + now := time.Now() + pods := []*corev1.Pod{ + stabilityTestPod("ready-idle", true, now), + stabilityTestPod("pending-idle", false, now), + stabilityTestPod("ready-allocated", true, now), + } + r := &PoolReconciler{} + if got := r.countReadyIdlePods(pods, []string{"ready-idle", "pending-idle"}); got != 1 { + t.Fatalf("countReadyIdlePods() = %d, want 1", got) + } +} + +func TestDesiredBufferCountUsesBandEdges(t *testing.T) { + tests := []struct { + name string + ready int32 + min int32 + max int32 + want int32 + }{ + {name: "below minimum", ready: 0, min: 20, max: 100, want: 20}, + {name: "at minimum", ready: 20, min: 20, max: 100, want: 20}, + {name: "inside band", ready: 89, min: 20, max: 100, want: 89}, + {name: "at maximum", ready: 100, min: 20, max: 100, want: 100}, + {name: "above maximum", ready: 120, min: 20, max: 100, want: 100}, + {name: "zero capacity", ready: 0, min: 0, max: 0, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := desiredBufferCount(tt.ready, tt.min, tt.max); got != tt.want { + t.Fatalf("desiredBufferCount(%d, %d, %d) = %d, want %d", tt.ready, tt.min, tt.max, got, tt.want) + } + }) + } +} + +func TestPickPodsToDeleteLeastUsefulFirst(t *testing.T) { + now := time.Now() + pendingOld := stabilityTestPod("pending-old", false, now.Add(-4*time.Minute)) + pendingNew := stabilityTestPod("pending-new", false, now.Add(-time.Minute)) + readyOld := stabilityTestPod("ready-old", true, now.Add(-5*time.Minute)) + readyNew := stabilityTestPod("ready-new", true, now.Add(-2*time.Minute)) + terminating := stabilityTestPod("terminating", false, now) + deletionTime := metav1.NewTime(now) + terminating.DeletionTimestamp = &deletionTime + + r := &PoolReconciler{} + got := r.pickPodsToDelete( + []*corev1.Pod{pendingOld, pendingNew, readyOld, readyNew, terminating}, + []string{pendingOld.Name, pendingNew.Name, readyOld.Name, readyNew.Name, terminating.Name}, + []string{readyNew.Name, readyNew.Name, terminating.Name}, + 3, + ) + + want := []string{"ready-new", "pending-new", "pending-old", "ready-old"} + if len(got) != len(want) { + t.Fatalf("pickPodsToDelete() returned %d pods, want %d", len(got), len(want)) + } + for i := range want { + if got[i].Name != want[i] { + t.Errorf("pod[%d] = %q, want %q", i, got[i].Name, want[i]) + } + } +} + +func TestScalePoolPendingPodsCoverDesiredCapacity(t *testing.T) { + PoolScaleExpectations = expectations.NewScaleExpectations() + t.Cleanup(func() { PoolScaleExpectations = expectations.NewScaleExpectations() }) + + now := time.Now() + pods := []*corev1.Pod{ + stabilityTestPod("pending-1", false, now), + stabilityTestPod("pending-2", false, now), + stabilityTestPod("pending-3", false, now), + stabilityTestPod("pending-4", false, now), + } + objects := make([]client.Object, 0, len(pods)) + idleNames := make([]string, 0, len(pods)) + for _, pod := range pods { + objects = append(objects, pod) + idleNames = append(idleNames, pod.Name) + } + r := stabilityTestReconciler(t, objects...) + pool := stabilityTestPool(intstr.FromString("100%")) + pool.Spec.CapacitySpec.BufferMin = 2 + pool.Spec.CapacitySpec.BufferMax = 4 + + pending, err := r.scalePool(context.Background(), pool, &scaleArgs{ + pods: pods, totalPodCnt: 4, idlePods: idleNames, supplyCnt: 2, + }) + if err != nil || pending { + t.Fatalf("scalePool() = pending %v, error %v", pending, err) + } + + var podList corev1.PodList + if err := r.List(context.Background(), &podList); err != nil { + t.Fatal(err) + } + if len(podList.Items) != 4 { + t.Fatalf("pod count = %d, want 4; pending capacity should prevent repeated creation", len(podList.Items)) + } +} + +func TestScalePoolCountsTerminatingPodsAgainstPoolMax(t *testing.T) { + PoolScaleExpectations = expectations.NewScaleExpectations() + t.Cleanup(func() { PoolScaleExpectations = expectations.NewScaleExpectations() }) + + now := time.Now() + active := []*corev1.Pod{ + stabilityTestPod("pending-1", false, now), + stabilityTestPod("pending-2", false, now), + } + r := stabilityTestReconciler(t, active[0], active[1]) + pool := stabilityTestPool(intstr.FromString("100%")) + pool.Spec.CapacitySpec.PoolMax = 3 + + _, err := r.scalePool(context.Background(), pool, &scaleArgs{ + pods: active, totalPodCnt: 3, idlePods: []string{"pending-1", "pending-2"}, supplyCnt: 5, + }) + if err != nil { + t.Fatal(err) + } + + var podList corev1.PodList + if err := r.List(context.Background(), &podList); err != nil { + t.Fatal(err) + } + if len(podList.Items) != 2 { + t.Fatalf("active pod count = %d, want 2; terminating occupancy must block creation", len(podList.Items)) + } +} + +func TestScalePoolLimitsDeleteBatch(t *testing.T) { + tests := []struct { + name string + maxUnavailable intstr.IntOrString + poolMin int32 + wantDeleted int + }{ + {name: "integer", maxUnavailable: intstr.FromInt(2), poolMin: 0, wantDeleted: 2}, + {name: "percentage", maxUnavailable: intstr.FromString("40%"), poolMin: 5, wantDeleted: 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + PoolScaleExpectations = expectations.NewScaleExpectations() + t.Cleanup(func() { PoolScaleExpectations = expectations.NewScaleExpectations() }) + + now := time.Now() + pods := make([]*corev1.Pod, 0, 10) + objects := make([]client.Object, 0, 10) + idleNames := make([]string, 0, 10) + for i := 0; i < 10; i++ { + pod := stabilityTestPod("pod-"+string(rune('a'+i)), false, now.Add(time.Duration(i)*time.Second)) + pods = append(pods, pod) + objects = append(objects, pod) + idleNames = append(idleNames, pod.Name) + } + r := stabilityTestReconciler(t, objects...) + pool := stabilityTestPool(tt.maxUnavailable) + pool.Spec.CapacitySpec.PoolMin = tt.poolMin + + _, err := r.scalePool(context.Background(), pool, &scaleArgs{ + pods: pods, totalPodCnt: 10, idlePods: idleNames, + }) + if err != nil { + t.Fatal(err) + } + deleteExpectations := PoolScaleExpectations.GetExpectations(controllerutils.GetControllerKey(pool))[expectations.Delete] + if deleteExpectations.Len() != tt.wantDeleted { + t.Fatalf("delete expectations = %d, want %d", deleteExpectations.Len(), tt.wantDeleted) + } + }) + } +} + +func TestObserveDeletedPodsCompletesDeleteBatch(t *testing.T) { + PoolScaleExpectations = expectations.NewScaleExpectations() + t.Cleanup(func() { PoolScaleExpectations = expectations.NewScaleExpectations() }) + controllerKey := "default/pool" + PoolScaleExpectations.ExpectScale(controllerKey, expectations.Delete, "gone") + PoolScaleExpectations.ExpectScale(controllerKey, expectations.Delete, "terminating") + + observeDeletedPods(controllerKey, map[string]struct{}{"terminating": {}}) + dirty := PoolScaleExpectations.GetExpectations(controllerKey)[expectations.Delete] + if dirty.Has("gone") || !dirty.Has("terminating") { + t.Fatalf("unexpected remaining delete expectations: %v", dirty.List()) + } + + observeDeletedPods(controllerKey, map[string]struct{}{}) + if satisfied, _, _ := PoolScaleExpectations.SatisfiedExpectations(controllerKey); !satisfied { + t.Fatal("expected delete batch to be satisfied after all pods disappear") + } +} + +type deleteFailingClient struct { + client.Client +} + +func (c *deleteFailingClient) Delete(context.Context, client.Object, ...client.DeleteOption) error { + return errors.New("delete failed") +} + +func TestScalePoolRollsBackExpectationOnDeleteFailure(t *testing.T) { + PoolScaleExpectations = expectations.NewScaleExpectations() + t.Cleanup(func() { PoolScaleExpectations = expectations.NewScaleExpectations() }) + + pod := stabilityTestPod("pod", false, time.Now()) + r := stabilityTestReconciler(t, pod) + r.Client = &deleteFailingClient{Client: r.Client} + pool := stabilityTestPool(intstr.FromInt(1)) + + _, err := r.scalePool(context.Background(), pool, &scaleArgs{ + pods: []*corev1.Pod{pod}, totalPodCnt: 1, idlePods: []string{pod.Name}, + }) + if err == nil { + t.Fatal("expected delete error") + } + if satisfied, _, dirty := PoolScaleExpectations.SatisfiedExpectations(controllerutils.GetControllerKey(pool)); !satisfied { + t.Fatalf("delete failure left stale expectation: %v", dirty) + } +} + +func TestScalePoolUnsatisfiedExpectationIsNotAnError(t *testing.T) { + PoolScaleExpectations = expectations.NewScaleExpectations() + t.Cleanup(func() { PoolScaleExpectations = expectations.NewScaleExpectations() }) + pool := stabilityTestPool(intstr.FromInt(1)) + PoolScaleExpectations.ExpectScale(controllerutils.GetControllerKey(pool), expectations.Delete, "terminating") + + r := stabilityTestReconciler(t) + pending, err := r.scalePool(context.Background(), pool, &scaleArgs{}) + if err != nil || !pending { + t.Fatalf("scalePool() = pending %v, error %v; want a non-error retry", pending, err) + } +} + +type stabilityAllocator struct { + *stubAllocator +} + +func (a *stabilityAllocator) Schedule(context.Context, *AllocSpec) (*algorithm.AllocAction, error) { + return &algorithm.AllocAction{}, nil +} + +func TestReconcilePoolUpdatesStatusWhileScaleIsPending(t *testing.T) { + PoolScaleExpectations = expectations.NewScaleExpectations() + t.Cleanup(func() { PoolScaleExpectations = expectations.NewScaleExpectations() }) + + pool := stabilityTestPool(intstr.FromInt(1)) + pool.Generation = 7 + r := stabilityTestReconciler(t, pool) + r.Allocator = &stabilityAllocator{stubAllocator: &stubAllocator{podAllocation: map[string]string{}}} + PoolScaleExpectations.ExpectScale(controllerutils.GetControllerKey(pool), expectations.Delete, "terminating") + + result, err := r.reconcilePool(context.Background(), pool, nil, nil, 1) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != defaultRetryTime { + t.Fatalf("RequeueAfter = %v, want %v", result.RequeueAfter, defaultRetryTime) + } + + updated := &sandboxv1alpha1.Pool{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(pool), updated); err != nil { + t.Fatal(err) + } + if updated.Status.ObservedGeneration != pool.Generation { + t.Fatalf("ObservedGeneration = %d, want %d", updated.Status.ObservedGeneration, pool.Generation) + } +}