From 701c50abb5d05488738e2fb5e8d418ae37bb1d6b Mon Sep 17 00:00:00 2001 From: Lukasz Wojciechowski Date: Thu, 23 Apr 2026 18:12:11 +0200 Subject: [PATCH 1/6] kubelet: enhance TestReconcileState Enhance CPU manager's test of reconcileState function (the one that actuates allocated CPU sets in runtime). The improvement involves three elements: 1) verification if lastUpdateState contains expected values; 2) enabling verification of how the reconcile process completed for multiple containers, not just a single one; 3) extending test cases to cover the above changes and add a simple multi-container test. Signed-off-by: Lukasz Wojciechowski --- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 272 +++++++++++++----- 1 file changed, 207 insertions(+), 65 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index d409179575fed..670ff70e1007d 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -951,20 +951,22 @@ func TestReconcileState(t *testing.T) { nil) testCases := []struct { - description string - policy Policy - activePods []*v1.Pod - pspPS v1.PodStatus - pspFound bool - updateErr error - stAssignments state.ContainerCPUAssignments - stDefaultCPUSet cpuset.CPUSet - lastUpdateStAssignments state.ContainerCPUAssignments - lastUpdateStDefaultCPUSet cpuset.CPUSet - expectStAssignments state.ContainerCPUAssignments - expectStDefaultCPUSet cpuset.CPUSet - expectSucceededContainerName string - expectFailedContainerName string + description string + policy Policy + activePods []*v1.Pod + pspPS v1.PodStatus + pspFound bool + updateErr error + stAssignments state.ContainerCPUAssignments + stDefaultCPUSet cpuset.CPUSet + lastUpdateStAssignments state.ContainerCPUAssignments + lastUpdateStDefaultCPUSet cpuset.CPUSet + expectStAssignments state.ContainerCPUAssignments + expectStDefaultCPUSet cpuset.CPUSet + expectLastUpdateStAssignments state.ContainerCPUAssignments + expectLastUpdateStDefaultCPUSet cpuset.CPUSet + expectSucceededContainerName []string + expectFailedContainerName []string }{ { description: "cpu manager reconcile - no error", @@ -1010,9 +1012,15 @@ func TestReconcileState(t *testing.T) { "fakeContainerName": cpuset.New(1, 2), }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodUID": map[string]cpuset.CPUSet{ + "fakeContainerName": cpuset.New(1, 2), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile init container - no error", @@ -1058,9 +1066,15 @@ func TestReconcileState(t *testing.T) { "fakeContainerName": cpuset.New(1, 2), }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodUID": map[string]cpuset.CPUSet{ + "fakeContainerName": cpuset.New(1, 2), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - pod status not found", @@ -1080,17 +1094,19 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspPS: v1.PodStatus{}, - pspFound: false, - updateErr: nil, - stAssignments: state.ContainerCPUAssignments{}, - stDefaultCPUSet: cpuset.New(), - lastUpdateStAssignments: state.ContainerCPUAssignments{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAssignments: state.ContainerCPUAssignments{}, - expectStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: "", - expectFailedContainerName: "", + pspPS: v1.PodStatus{}, + pspFound: false, + updateErr: nil, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(), + lastUpdateStAssignments: state.ContainerCPUAssignments{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{}, + expectStDefaultCPUSet: cpuset.New(), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - container state not found", @@ -1118,16 +1134,18 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, - stAssignments: state.ContainerCPUAssignments{}, - stDefaultCPUSet: cpuset.New(), - lastUpdateStAssignments: state.ContainerCPUAssignments{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAssignments: state.ContainerCPUAssignments{}, - expectStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: "", - expectFailedContainerName: "fakeContainerName", + pspFound: true, + updateErr: nil, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(), + lastUpdateStAssignments: state.ContainerCPUAssignments{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{}, + expectStDefaultCPUSet: cpuset.New(), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconclie - cpuset is empty", @@ -1173,9 +1191,11 @@ func TestReconcileState(t *testing.T) { "fakeContainerName": cpuset.New(), }, }, - expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), - expectSucceededContainerName: "", - expectFailedContainerName: "fakeContainerName", + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerName"}, }, { description: "cpu manager reconclie - container update error", @@ -1221,9 +1241,11 @@ func TestReconcileState(t *testing.T) { "fakeContainerName": cpuset.New(1, 2), }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "", - expectFailedContainerName: "fakeContainerName", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerName"}, }, { description: "cpu manager reconcile - state has inactive container", @@ -1272,9 +1294,15 @@ func TestReconcileState(t *testing.T) { "fakeContainerName": cpuset.New(1, 2), }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodUID": map[string]cpuset.CPUSet{ + "fakeContainerName": cpuset.New(1, 2), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is current", @@ -1318,15 +1346,21 @@ func TestReconcileState(t *testing.T) { "fakeContainerName": cpuset.New(1, 2), }, }, - lastUpdateStDefaultCPUSet: cpuset.New(5, 6, 7), + lastUpdateStDefaultCPUSet: cpuset.New(), expectStAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), }, }, - expectStDefaultCPUSet: cpuset.New(5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodUID": map[string]cpuset.CPUSet{ + "fakeContainerName": cpuset.New(1, 2), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is not current", @@ -1370,15 +1404,113 @@ func TestReconcileState(t *testing.T) { "fakeContainerName": cpuset.New(3, 4), }, }, - lastUpdateStDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), + lastUpdateStDefaultCPUSet: cpuset.New(), expectStAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodUID": map[string]cpuset.CPUSet{ + "fakeContainerName": cpuset.New(1, 2), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, + }, + { + description: "cpu manager reconcile default CPU sets - no error", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + lastUpdateStAssignments: state.ContainerCPUAssignments{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{}, + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(1, 2, 3, 4, 5, 6, 7), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(1, 2, 3, 4, 5, 6, 7), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(1, 2, 3, 4, 5, 6, 7), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, }, } @@ -1404,47 +1536,57 @@ func TestReconcileState(t *testing.T) { }, } mgr.sourcesReady = &sourcesReadyStub{} + mgr.lastUpdateState.SetCPUAssignments(testCase.lastUpdateStAssignments) + mgr.lastUpdateState.SetDefaultCPUSet(testCase.lastUpdateStDefaultCPUSet) success, failure := mgr.reconcileState(context.Background()) if !reflect.DeepEqual(testCase.expectStAssignments, mgr.state.GetCPUAssignments()) { t.Errorf("%v", testCase.description) t.Errorf("Expected state container cpu assignments: %v, actual: %v", testCase.expectStAssignments, mgr.state.GetCPUAssignments()) - } if !reflect.DeepEqual(testCase.expectStDefaultCPUSet, mgr.state.GetDefaultCPUSet()) { t.Errorf("%v", testCase.description) t.Errorf("Expected state default cpuset: %v, actual: %v", testCase.expectStDefaultCPUSet, mgr.state.GetDefaultCPUSet()) + } + + if !reflect.DeepEqual(testCase.expectLastUpdateStAssignments, mgr.lastUpdateState.GetCPUAssignments()) { + t.Errorf("%v", testCase.description) + t.Errorf("Expected lastUpdateState container cpu assignments: %v, actual: %v", testCase.expectLastUpdateStAssignments, mgr.lastUpdateState.GetCPUAssignments()) + } + if !reflect.DeepEqual(testCase.expectLastUpdateStDefaultCPUSet, mgr.lastUpdateState.GetDefaultCPUSet()) { + t.Errorf("%v", testCase.description) + t.Errorf("Expected lastUpdateState default cpuset: %v, actual: %v", testCase.expectLastUpdateStDefaultCPUSet, mgr.lastUpdateState.GetDefaultCPUSet()) } - if testCase.expectSucceededContainerName != "" { + for _, name := range testCase.expectSucceededContainerName { // Search succeeded reconciled containers for the supplied name. foundSucceededContainer := false for _, reconciled := range success { - if reconciled.containerName == testCase.expectSucceededContainerName { + if reconciled.containerName == name { foundSucceededContainer = true break } } if !foundSucceededContainer { t.Errorf("%v", testCase.description) - t.Errorf("Expected reconciliation success for container: %s", testCase.expectSucceededContainerName) + t.Errorf("Expected reconciliation success for container: %s", name) } } - if testCase.expectFailedContainerName != "" { + for _, name := range testCase.expectFailedContainerName { // Search failed reconciled containers for the supplied name. foundFailedContainer := false for _, reconciled := range failure { - if reconciled.containerName == testCase.expectFailedContainerName { + if reconciled.containerName == name { foundFailedContainer = true break } } if !foundFailedContainer { t.Errorf("%v", testCase.description) - t.Errorf("Expected reconciliation failure for container: %s", testCase.expectFailedContainerName) + t.Errorf("Expected reconciliation failure for container: %s", name) } } } From 0204de816370b5313c456c7f57931c6389f63dd8 Mon Sep 17 00:00:00 2001 From: Lukasz Wojciechowski Date: Sat, 25 Apr 2026 02:41:48 +0200 Subject: [PATCH 2/6] kubelet: enhance TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs Enhance CPU manager's test of reconcileState function for feature InPlacePodVerticalScalingExclusiveCPUs enabled. This is similar changes that one made to TestReconcileState. The improvement involves three elements: 1) verification if lastUpdateState contains expected values; 2) enabling verification of how the reconcile process completed for multiple containers, not just a single one; 3) extending test cases to cover the above changes and add a simple multi-container test. Signed-off-by: Lukasz Wojciechowski --- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 272 +++++++++++++----- 1 file changed, 208 insertions(+), 64 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index 670ff70e1007d..58fba360304af 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -2355,20 +2355,22 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) nil) testCases := []struct { - description string - policy Policy - activePods []*v1.Pod - pspPS v1.PodStatus - pspFound bool - updateErr error - stAllocations state.ContainerCPUAllocations - stDefaultCPUSet cpuset.CPUSet - lastUpdateStAllocations state.ContainerCPUAllocations - lastUpdateStDefaultCPUSet cpuset.CPUSet - expectStAllocations state.ContainerCPUAllocations - expectStDefaultCPUSet cpuset.CPUSet - expectSucceededContainerName string - expectFailedContainerName string + description string + policy Policy + activePods []*v1.Pod + pspPS v1.PodStatus + pspFound bool + updateErr error + stAllocations state.ContainerCPUAllocations + stDefaultCPUSet cpuset.CPUSet + lastUpdateStAllocations state.ContainerCPUAllocations + lastUpdateStDefaultCPUSet cpuset.CPUSet + expectStAllocations state.ContainerCPUAllocations + expectStDefaultCPUSet cpuset.CPUSet + expectLastUpdateStAllocations state.ContainerCPUAllocations + expectLastUpdateStDefaultCPUSet cpuset.CPUSet + expectSucceededContainerName []string + expectFailedContainerName []string }{ { description: "cpu manager reconcile - no error", @@ -2414,9 +2416,15 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile init container - no error", @@ -2462,9 +2470,15 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - pod status not found", @@ -2484,17 +2498,19 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspPS: v1.PodStatus{}, - pspFound: false, - updateErr: nil, - stAllocations: state.ContainerCPUAllocations{}, - stDefaultCPUSet: cpuset.New(), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAllocations: state.ContainerCPUAllocations{}, - expectStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: "", - expectFailedContainerName: "", + pspPS: v1.PodStatus{}, + pspFound: false, + updateErr: nil, + stAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{}, + expectStDefaultCPUSet: cpuset.New(), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - container state not found", @@ -2522,16 +2538,18 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, - stAllocations: state.ContainerCPUAllocations{}, - stDefaultCPUSet: cpuset.New(), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAllocations: state.ContainerCPUAllocations{}, - expectStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: "", - expectFailedContainerName: "fakeContainerName", + pspFound: true, + updateErr: nil, + stAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{}, + expectStDefaultCPUSet: cpuset.New(), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconclie - cpuset is empty", @@ -2577,9 +2595,11 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) "fakeContainerName": {Original: cpuset.New(), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), - expectSucceededContainerName: "", - expectFailedContainerName: "fakeContainerName", + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerName"}, }, { description: "cpu manager reconclie - container update error", @@ -2625,9 +2645,11 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "", - expectFailedContainerName: "fakeContainerName", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerName"}, }, { description: "cpu manager reconcile - state has inactive container", @@ -2676,9 +2698,15 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is current", @@ -2722,15 +2750,21 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - lastUpdateStDefaultCPUSet: cpuset.New(5, 6, 7), + lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is not current", @@ -2774,15 +2808,113 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) "fakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, }, }, - lastUpdateStDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), + lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectSucceededContainerName: "fakeContainerName", - expectFailedContainerName: "", + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New(1, 2)}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, + }, + { + description: "cpu manager reconcile default CPU sets - no error", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + stAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{}, + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, }, } @@ -2808,6 +2940,8 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, } mgr.sourcesReady = &sourcesReadyStub{} + mgr.lastUpdateState.SetCPUAllocations(testCase.lastUpdateStAllocations) + mgr.lastUpdateState.SetDefaultCPUSet(testCase.lastUpdateStDefaultCPUSet) success, failure := mgr.reconcileState(context.Background()) if !reflect.DeepEqual(testCase.expectStAllocations, mgr.state.GetCPUAllocations()) { @@ -2822,33 +2956,43 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) } - if testCase.expectSucceededContainerName != "" { + if !reflect.DeepEqual(testCase.expectLastUpdateStAllocations, mgr.lastUpdateState.GetCPUAllocations()) { + t.Errorf("%v", testCase.description) + t.Errorf("Expected lastUpdateState container cpu allocations: %v, actual: %v", testCase.expectLastUpdateStAllocations, mgr.lastUpdateState.GetCPUAllocations()) + } + + if !reflect.DeepEqual(testCase.expectLastUpdateStDefaultCPUSet, mgr.lastUpdateState.GetDefaultCPUSet()) { + t.Errorf("%v", testCase.description) + t.Errorf("Expected lastUpdateState default cpuset: %v, actual: %v", testCase.expectLastUpdateStDefaultCPUSet, mgr.lastUpdateState.GetDefaultCPUSet()) + } + + for _, name := range testCase.expectSucceededContainerName { // Search succeeded reconciled containers for the supplied name. foundSucceededContainer := false for _, reconciled := range success { - if reconciled.containerName == testCase.expectSucceededContainerName { + if reconciled.containerName == name { foundSucceededContainer = true break } } if !foundSucceededContainer { t.Errorf("%v", testCase.description) - t.Errorf("Expected reconciliation success for container: %s", testCase.expectSucceededContainerName) + t.Errorf("Expected reconciliation success for container: %s", name) } } - if testCase.expectFailedContainerName != "" { + for _, name := range testCase.expectFailedContainerName { // Search failed reconciled containers for the supplied name. foundFailedContainer := false for _, reconciled := range failure { - if reconciled.containerName == testCase.expectFailedContainerName { + if reconciled.containerName == name { foundFailedContainer = true break } } if !foundFailedContainer { t.Errorf("%v", testCase.description) - t.Errorf("Expected reconciliation failure for container: %s", testCase.expectFailedContainerName) + t.Errorf("Expected reconciliation failure for container: %s", name) } } } From 916ddf87ae58c55149b8622e5b6d909aab20cc62 Mon Sep 17 00:00:00 2001 From: Lukasz Wojciechowski Date: Sat, 25 Apr 2026 03:43:21 +0200 Subject: [PATCH 3/6] kubelet: verify CPUSets set in mock runtime Extend mockRuntimeService in the CPU manager tests to store CPUSets that were applied to the runtime. Additionally, after each update of container resources (if the testCPUConflicts flag is enabled), the mock runtime verifies if exclusive CPUs are assigned to a single container only. The extended mockRuntimeService is applied to TestReconcileState and TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs, so it verifies conflicts after each update and after reconciliation is completed, it is verified if the runtime state matches expectations. Proper fields are added to each of the existing test cases. Additionally, the mockRuntimeService has been enhanced to support returning a sequence of errors from UpdateContainerResources calls. The err field was changed from a single error to a slice of errors ([]error), and the UpdateContainerResources function now returns the first error from the slice and removes it, allowing different errors to be returned for successive calls. This enhancement enables more sophisticated testing scenarios where multiple container resource updates may fail at different points in the test sequence. Proper fields are added to each of the existing test cases to accommodate the new error slice functionality and CPU set tracking capabilities. Signed-off-by: Lukasz Wojciechowski --- .../cm/cpumanager/cpu_manager_others_test.go | 36 ++ pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 351 +++++++++++++----- .../cm/cpumanager/cpu_manager_windows_test.go | 42 +++ .../cpumanager/policy_static_restore_test.go | 4 +- 4 files changed, 335 insertions(+), 98 deletions(-) create mode 100644 pkg/kubelet/cm/cpumanager/cpu_manager_others_test.go create mode 100644 pkg/kubelet/cm/cpumanager/cpu_manager_windows_test.go diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_others_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_others_test.go new file mode 100644 index 0000000000000..07a1406177b6f --- /dev/null +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_others_test.go @@ -0,0 +1,36 @@ +//go:build !windows + +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cpumanager + +import ( + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "k8s.io/utils/cpuset" +) + +func (rt mockRuntimeService) getCPUSetFromResources(resources *runtimeapi.ContainerResources) cpuset.CPUSet { + if resources != nil && resources.Linux != nil { + set, err := cpuset.Parse(resources.Linux.CpusetCpus) + if err != nil { + rt.t.Errorf("(%v) Cannot parse Linux CPUSet resources %v", rt.testCaseDescription, resources.Linux.CpusetCpus) + return cpuset.New() + } + return set + } + return cpuset.New() +} diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index 58fba360304af..339679a2c924d 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -218,15 +218,52 @@ func (p *mockPolicy) GetAllocatableCPUs(m state.State) cpuset.CPUSet { } type mockRuntimeService struct { - err error + err []error + containerIDsWithExclusiveCPUs []string + state map[string]cpuset.CPUSet + testCPUConflicts bool + testCaseDescription string + t *testing.T } -func (rt mockRuntimeService) UpdateContainerResources(_ context.Context, id string, resources *runtimeapi.ContainerResources) error { - return rt.err -} +func (rt *mockRuntimeService) UpdateContainerResources(_ context.Context, id string, resources *runtimeapi.ContainerResources) error { + var ret error + if len(rt.err) > 0 { + ret = rt.err[0] + rt.err = rt.err[1:] + } + + // update state + if ret == nil { + newSet := rt.getCPUSetFromResources(resources) + if !newSet.IsEmpty() { + rt.state[id] = newSet + } + } + + if rt.testCPUConflicts { + // count in how many containers each CPU is used + cpuUsage := make(map[int][]string) + for containerID, set := range rt.state { + for _, cpu := range set.List() { + cpuUsage[cpu] = append(cpuUsage[cpu], containerID) + } + } + + // check if CPUs assigned to containers with exclusive CPUs are used exactly once + for _, containerID := range rt.containerIDsWithExclusiveCPUs { + set := rt.state[containerID] + for _, cpu := range set.List() { + if len(cpuUsage[cpu]) != 1 { + rt.t.Errorf("%v", rt.testCaseDescription) + rt.t.Errorf("after updating container resources of %s", id) + rt.t.Errorf("Expected CPU %d usage 1, actual usage %d %v", cpu, len(cpuUsage[cpu]), cpuUsage[cpu]) + } + } + } + } -func (rt mockRuntimeService) Close(_ context.Context) error { - return rt.err + return ret } type mockPodStatusProvider struct { @@ -434,7 +471,7 @@ func TestCPUManagerAdd(t *testing.T) { nil) testCases := []struct { description string - updateErr error + updateErr []error policy Policy expCPUSet cpuset.CPUSet expAllocateErr error @@ -468,7 +505,7 @@ func TestCPUManagerAdd(t *testing.T) { defaultCPUSet: cpuset.New(1, 2, 3, 4), }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{ + containerRuntime: &mockRuntimeService{ err: testCase.updateErr, }, containerMap: containermap.NewContainerMap(), @@ -698,7 +735,7 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) { policy: policy, state: mockState, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{}, + containerRuntime: &mockRuntimeService{}, containerMap: containermap.NewContainerMap(), podStatusProvider: mockPodStatusProvider{}, sourcesReady: &sourcesReadyStub{}, @@ -890,7 +927,7 @@ func TestCPUManagerRemove(t *testing.T) { defaultCPUSet: cpuset.New(), }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{}, + containerRuntime: &mockRuntimeService{}, containerMap: containerMap, activePods: func() []*v1.Pod { return nil }, podStatusProvider: mockPodStatusProvider{}, @@ -907,7 +944,7 @@ func TestCPUManagerRemove(t *testing.T) { err: fmt.Errorf("fake error"), }, state: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{}, + containerRuntime: &mockRuntimeService{}, containerMap: containerMap, activePods: func() []*v1.Pod { return nil }, podStatusProvider: mockPodStatusProvider{}, @@ -956,7 +993,9 @@ func TestReconcileState(t *testing.T) { activePods []*v1.Pod pspPS v1.PodStatus pspFound bool - updateErr error + updateErr []error + containerIDsWithExclusiveCPUs []string + containerRuntimeInitialState map[string]cpuset.CPUSet stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet lastUpdateStAssignments state.ContainerCPUAssignments @@ -965,6 +1004,7 @@ func TestReconcileState(t *testing.T) { expectStDefaultCPUSet cpuset.CPUSet expectLastUpdateStAssignments state.ContainerCPUAssignments expectLastUpdateStDefaultCPUSet cpuset.CPUSet + expectContainerRuntimeState map[string]cpuset.CPUSet expectSucceededContainerName []string expectFailedContainerName []string }{ @@ -997,8 +1037,10 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), @@ -1019,8 +1061,11 @@ func TestReconcileState(t *testing.T) { }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile init container - no error", @@ -1051,8 +1096,10 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), @@ -1073,8 +1120,11 @@ func TestReconcileState(t *testing.T) { }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - pod status not found", @@ -1097,6 +1147,8 @@ func TestReconcileState(t *testing.T) { pspPS: v1.PodStatus{}, pspFound: false, updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(), lastUpdateStAssignments: state.ContainerCPUAssignments{}, @@ -1105,6 +1157,7 @@ func TestReconcileState(t *testing.T) { expectStDefaultCPUSet: cpuset.New(), expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{}, }, @@ -1136,6 +1189,8 @@ func TestReconcileState(t *testing.T) { }, pspFound: true, updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(), lastUpdateStAssignments: state.ContainerCPUAssignments{}, @@ -1144,6 +1199,7 @@ func TestReconcileState(t *testing.T) { expectStDefaultCPUSet: cpuset.New(), expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{}, }, @@ -1176,8 +1232,10 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(), @@ -1194,6 +1252,7 @@ func TestReconcileState(t *testing.T) { expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{"fakeContainerName"}, }, @@ -1226,8 +1285,10 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: fmt.Errorf("fake container update error"), + pspFound: true, + updateErr: []error{fmt.Errorf("fake container update error")}, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), @@ -1244,6 +1305,7 @@ func TestReconcileState(t *testing.T) { expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), expectLastUpdateStAssignments: state.ContainerCPUAssignments{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{"fakeContainerName"}, }, @@ -1276,8 +1338,10 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), @@ -1301,8 +1365,11 @@ func TestReconcileState(t *testing.T) { }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is current", @@ -1333,8 +1400,12 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, stAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), @@ -1359,8 +1430,11 @@ func TestReconcileState(t *testing.T) { }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is not current", @@ -1391,8 +1465,12 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(3, 4), + }, stAssignments: state.ContainerCPUAssignments{ "fakePodUID": map[string]cpuset.CPUSet{ "fakeContainerName": cpuset.New(1, 2), @@ -1417,11 +1495,14 @@ func TestReconcileState(t *testing.T) { }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { - description: "cpu manager reconcile default CPU sets - no error", + description: "cpu manager reconcile - default CPU sets no error", policy: testPolicy, activePods: []*v1.Pod{ { @@ -1489,14 +1570,16 @@ func TestReconcileState(t *testing.T) { }, }, }, - pspFound: true, - updateErr: nil, - stAssignments: state.ContainerCPUAssignments{}, - stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), - lastUpdateStAssignments: state.ContainerCPUAssignments{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAssignments: state.ContainerCPUAssignments{}, - expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + lastUpdateStAssignments: state.ContainerCPUAssignments{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{}, + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), expectLastUpdateStAssignments: state.ContainerCPUAssignments{ "fakePodAUID": map[string]cpuset.CPUSet{ "fakeContainerAName": cpuset.New(1, 2, 3, 4, 5, 6, 7), @@ -1509,8 +1592,13 @@ func TestReconcileState(t *testing.T) { }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + "fakeContainerBID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + "fakeContainerCID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, }, } @@ -1523,8 +1611,13 @@ func TestReconcileState(t *testing.T) { defaultCPUSet: testCase.stDefaultCPUSet, }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{ - err: testCase.updateErr, + containerRuntime: &mockRuntimeService{ + err: testCase.updateErr, + containerIDsWithExclusiveCPUs: testCase.containerIDsWithExclusiveCPUs, + state: testCase.containerRuntimeInitialState, + testCPUConflicts: true, + testCaseDescription: testCase.description, + t: t, }, containerMap: containermap.NewContainerMap(), activePods: func() []*v1.Pod { @@ -1560,6 +1653,11 @@ func TestReconcileState(t *testing.T) { t.Errorf("Expected lastUpdateState default cpuset: %v, actual: %v", testCase.expectLastUpdateStDefaultCPUSet, mgr.lastUpdateState.GetDefaultCPUSet()) } + if !reflect.DeepEqual(testCase.expectContainerRuntimeState, mgr.containerRuntime.(*mockRuntimeService).state) { + t.Errorf("%v", testCase.description) + t.Errorf("Expected containerRuntimeState: %v, actual: %v", testCase.expectContainerRuntimeState, mgr.containerRuntime.(*mockRuntimeService).state) + } + for _, name := range testCase.expectSucceededContainerName { // Search succeeded reconciled containers for the supplied name. foundSucceededContainer := false @@ -1620,7 +1718,7 @@ func TestCPUManagerAddWithResvList(t *testing.T) { nil) testCases := []struct { description string - updateErr error + updateErr []error policy Policy expCPUSet cpuset.CPUSet expAllocateErr error @@ -1644,7 +1742,7 @@ func TestCPUManagerAddWithResvList(t *testing.T) { defaultCPUSet: cpuset.New(0, 1, 2, 3), }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{ + containerRuntime: &mockRuntimeService{ err: testCase.updateErr, }, containerMap: containermap.NewContainerMap(), @@ -1844,7 +1942,7 @@ func TestCPUManagerAddWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) { nil) testCases := []struct { description string - updateErr error + updateErr []error policy Policy expCPUSet cpuset.CPUSet expAllocateErr error @@ -1878,7 +1976,7 @@ func TestCPUManagerAddWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) { defaultCPUSet: cpuset.New(1, 2, 3, 4), }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{ + containerRuntime: &mockRuntimeService{ err: testCase.updateErr, }, containerMap: containermap.NewContainerMap(), @@ -2108,7 +2206,7 @@ func TestCPUManagerAddWithInitContainersWithInPlacePodVerticalScalingExclusiveCP policy: policy, state: mockState, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{}, + containerRuntime: &mockRuntimeService{}, containerMap: containermap.NewContainerMap(), podStatusProvider: mockPodStatusProvider{}, sourcesReady: &sourcesReadyStub{}, @@ -2294,7 +2392,7 @@ func TestCPUManagerRemoveWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T defaultCPUSet: cpuset.New(), }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{}, + containerRuntime: &mockRuntimeService{}, containerMap: containerMap, activePods: func() []*v1.Pod { return nil }, podStatusProvider: mockPodStatusProvider{}, @@ -2311,7 +2409,7 @@ func TestCPUManagerRemoveWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T err: fmt.Errorf("fake error"), }, state: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{}, + containerRuntime: &mockRuntimeService{}, containerMap: containerMap, activePods: func() []*v1.Pod { return nil }, podStatusProvider: mockPodStatusProvider{}, @@ -2360,7 +2458,9 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) activePods []*v1.Pod pspPS v1.PodStatus pspFound bool - updateErr error + updateErr []error + containerIDsWithExclusiveCPUs []string + containerRuntimeInitialState map[string]cpuset.CPUSet stAllocations state.ContainerCPUAllocations stDefaultCPUSet cpuset.CPUSet lastUpdateStAllocations state.ContainerCPUAllocations @@ -2369,6 +2469,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectStDefaultCPUSet cpuset.CPUSet expectLastUpdateStAllocations state.ContainerCPUAllocations expectLastUpdateStDefaultCPUSet cpuset.CPUSet + expectContainerRuntimeState map[string]cpuset.CPUSet expectSucceededContainerName []string expectFailedContainerName []string }{ @@ -2401,8 +2502,10 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, @@ -2423,8 +2526,11 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile init container - no error", @@ -2455,8 +2561,10 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, @@ -2477,8 +2585,11 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - pod status not found", @@ -2501,6 +2612,8 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspPS: v1.PodStatus{}, pspFound: false, updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAllocations: state.ContainerCPUAllocations{}, stDefaultCPUSet: cpuset.New(), lastUpdateStAllocations: state.ContainerCPUAllocations{}, @@ -2509,6 +2622,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectStDefaultCPUSet: cpuset.New(), expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{}, }, @@ -2540,6 +2654,8 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, pspFound: true, updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAllocations: state.ContainerCPUAllocations{}, stDefaultCPUSet: cpuset.New(), lastUpdateStAllocations: state.ContainerCPUAllocations{}, @@ -2548,6 +2664,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectStDefaultCPUSet: cpuset.New(), expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{}, }, @@ -2580,8 +2697,10 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(), Resized: cpuset.New()}, @@ -2598,6 +2717,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{"fakeContainerName"}, }, @@ -2630,8 +2750,10 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: fmt.Errorf("fake container update error"), + pspFound: true, + updateErr: []error{fmt.Errorf("fake container update error")}, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, @@ -2648,6 +2770,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, expectSucceededContainerName: []string{}, expectFailedContainerName: []string{"fakeContainerName"}, }, @@ -2680,8 +2803,10 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, @@ -2705,8 +2830,11 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is current", @@ -2737,8 +2865,12 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, @@ -2763,8 +2895,11 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { description: "cpu manager reconcile - last update state is not current", @@ -2795,8 +2930,12 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(3, 4), + }, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, @@ -2821,11 +2960,14 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { - description: "cpu manager reconcile default CPU sets - no error", + description: "cpu manager reconcile - default CPU sets no error", policy: testPolicy, activePods: []*v1.Pod{ { @@ -2893,14 +3035,16 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, - stAllocations: state.ContainerCPUAllocations{}, - stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAllocations: state.ContainerCPUAllocations{}, - expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{}, + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ "fakeContainerAName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, @@ -2913,8 +3057,13 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + "fakeContainerBID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + "fakeContainerCID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, }, } @@ -2927,8 +3076,13 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) defaultCPUSet: testCase.stDefaultCPUSet, }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{ - err: testCase.updateErr, + containerRuntime: &mockRuntimeService{ + err: testCase.updateErr, + containerIDsWithExclusiveCPUs: testCase.containerIDsWithExclusiveCPUs, + state: testCase.containerRuntimeInitialState, + testCPUConflicts: true, + testCaseDescription: testCase.description, + t: t, }, containerMap: containermap.NewContainerMap(), activePods: func() []*v1.Pod { @@ -2966,6 +3120,11 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) t.Errorf("Expected lastUpdateState default cpuset: %v, actual: %v", testCase.expectLastUpdateStDefaultCPUSet, mgr.lastUpdateState.GetDefaultCPUSet()) } + if !reflect.DeepEqual(testCase.expectContainerRuntimeState, mgr.containerRuntime.(*mockRuntimeService).state) { + t.Errorf("%v", testCase.description) + t.Errorf("Expected containerRuntimeState: %v, actual: %v", testCase.expectContainerRuntimeState, mgr.containerRuntime.(*mockRuntimeService).state) + } + for _, name := range testCase.expectSucceededContainerName { // Search succeeded reconciled containers for the supplied name. foundSucceededContainer := false @@ -3026,7 +3185,7 @@ func TestCPUManagerAddWithResvListWithInPlacePodVerticalScalingExclusiveCPUs(t * nil) testCases := []struct { description string - updateErr error + updateErr []error policy Policy expCPUSet cpuset.CPUSet expAllocateErr error @@ -3050,7 +3209,7 @@ func TestCPUManagerAddWithResvListWithInPlacePodVerticalScalingExclusiveCPUs(t * defaultCPUSet: cpuset.New(0, 1, 2, 3), }, lastUpdateState: state.NewMemoryState(logger), - containerRuntime: mockRuntimeService{ + containerRuntime: &mockRuntimeService{ err: testCase.updateErr, }, containerMap: containermap.NewContainerMap(), diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_windows_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_windows_test.go new file mode 100644 index 0000000000000..ccc1447f76394 --- /dev/null +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_windows_test.go @@ -0,0 +1,42 @@ +//go:build windows + +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cpumanager + +import ( + utilfeature "k8s.io/apiserver/pkg/util/feature" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + kubefeatures "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/kubelet/winstats" + "k8s.io/utils/cpuset" +) + +func (rt mockRuntimeService) getCPUSetFromResources(resources *runtimeapi.ContainerResources) cpuset.CPUSet { + if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsCPUAndMemoryAffinity) { + return cpuset.New() + } + if resources != nil && resources.Windows != nil { + var cpus []int + for _, affinity := range resources.Windows.AffinityCpus { + ga := winstats.GroupAffinity{Mask: affinity.CpuMask, Group: uint16(affinity.CpuGroup)} + cpus = append(cpus, ga.Processors()...) + } + return cpuset.New(cpus...) + } + return cpuset.New() +} diff --git a/pkg/kubelet/cm/cpumanager/policy_static_restore_test.go b/pkg/kubelet/cm/cpumanager/policy_static_restore_test.go index 2476d44888b07..06a3f807f62b4 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static_restore_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_restore_test.go @@ -141,7 +141,7 @@ func TestCPUManagerRestoreState(t *testing.T) { pod.UID = types.UID("pod1") // Start manager to initialize state and activePods - err = mgr.Start(context.Background(), func() []*v1.Pod { return []*v1.Pod{pod} }, &sourcesReadyStub{}, mockPodStatusProvider{}, mockRuntimeService{}, containermap.NewContainerMap()) + err = mgr.Start(context.Background(), func() []*v1.Pod { return []*v1.Pod{pod} }, &sourcesReadyStub{}, mockPodStatusProvider{}, &mockRuntimeService{}, containermap.NewContainerMap()) if err != nil { t.Fatalf("could not start manager: %v", err) } @@ -205,7 +205,7 @@ func TestCPUManagerRestoreState(t *testing.T) { t.Fatalf("could not create manager 2: %v", err) } - err = mgr2.Start(context.Background(), func() []*v1.Pod { return []*v1.Pod{pod} }, &sourcesReadyStub{}, mockPodStatusProvider{}, mockRuntimeService{}, containermap.NewContainerMap()) + err = mgr2.Start(context.Background(), func() []*v1.Pod { return []*v1.Pod{pod} }, &sourcesReadyStub{}, mockPodStatusProvider{}, &mockRuntimeService{}, containermap.NewContainerMap()) if err != nil { t.Fatalf("could not start manager 2: %v", err) } From fe45d23a15feeb690cf06690bec4867850410351 Mon Sep 17 00:00:00 2001 From: Lukasz Wojciechowski Date: Sat, 25 Apr 2026 04:18:11 +0200 Subject: [PATCH 4/6] kubelet: add resize testcases for reconcileState Add test cases for verification of CPUSets reconcilation for containers using exclusive CPUs to TestCPUManagerAddWithResvListWithInPlacePodVerticalScalingExclusiveCPUs These test cases verify behavior of CPUs scaling with InPlacePodVerticalScalingExclusiveCPUs enabled. Signed-off-by: Lukasz Wojciechowski --- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 468 ++++++++++++++++++ 1 file changed, 468 insertions(+) diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index 339679a2c924d..2a60fb34c675d 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -3065,6 +3065,474 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, expectFailedContainerName: []string{}, }, + { + description: "cpu manager reconcile - exclusive cpu container scaled up", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerBID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 5, 6, 7), + "fakeContainerBID": cpuset.New(3, 4), + "fakeContainerCID": cpuset.New(1, 2, 5, 6, 7), + }, + stAllocations: state.ContainerCPUAllocations{ + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + }, + }, + stDefaultCPUSet: cpuset.New(1, 2, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New()}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New()}, + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + }, + }, + expectStDefaultCPUSet: cpuset.New(1, 2, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New(1, 2, 7)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New(1, 2, 7)}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 7), + "fakeContainerBID": cpuset.New(3, 4, 5, 6), + "fakeContainerCID": cpuset.New(1, 2, 7), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, + }, + { + description: "cpu manager reconcile - exclusive cpu container scaled down", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerBID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 7), + "fakeContainerBID": cpuset.New(3, 4, 5, 6), + "fakeContainerCID": cpuset.New(1, 2, 7), + }, + stAllocations: state.ContainerCPUAllocations{ + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4)}, + }, + }, + stDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New()}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4, 5, 6), Resized: cpuset.New()}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New()}, + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4)}, + }, + }, + expectStDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New(1, 2, 5, 6, 7)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4, 5, 6), Resized: cpuset.New(3, 4)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New(1, 2, 5, 6, 7)}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 5, 6, 7), + "fakeContainerBID": cpuset.New(3, 4), + "fakeContainerCID": cpuset.New(1, 2, 5, 6, 7), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, + }, + { + description: "cpu manager reconcile - exclusive cpu containers swap CPUs", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerBID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2), + "fakeContainerBID": cpuset.New(3, 4), + }, + stAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, + }, + }, + stDefaultCPUSet: cpuset.New(5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3, 4)}, + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, + }, + }, + expectStDefaultCPUSet: cpuset.New(5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 4), + "fakeContainerBID": cpuset.New(2, 3), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName"}, + expectFailedContainerName: []string{}, + }, + { + description: "cpu manager reconcile - exclusive cpu containers scaled down and up", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerBID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2), + "fakeContainerBID": cpuset.New(3, 4), + "fakeContainerCID": cpuset.New(5, 6), + }, + stAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, + }, + }, + stDefaultCPUSet: cpuset.New(4), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3, 4)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(5, 6), Resized: cpuset.New()}, + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, + }, + }, + expectStDefaultCPUSet: cpuset.New(4), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(5, 6), Resized: cpuset.New(4)}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 5, 6), + "fakeContainerBID": cpuset.New(3), + "fakeContainerCID": cpuset.New(4), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, + }, } for _, testCase := range testCases { From 5f69c1dcda03d16a31581a9ae37e80ea41f6d101 Mon Sep 17 00:00:00 2001 From: Lukasz Wojciechowski Date: Fri, 6 Mar 2026 18:03:15 +0100 Subject: [PATCH 5/6] kubelet: rework CPUSet reconciliation algorithm The former implementation of reconcileState in CPU manager had two issues: 1) It didn't apply CPU sets of all containers as a consistent state. The loop for all pods and containers applied CPUSets one by one without any critical section. During iteration over loop, the allocated CPUSets and default set could have been changed by executing Allocate for needs of resize or appearance of new container. Such situation could lead to conflicts of exclusive CPUs, e.g. a) reconcileState applies default CPU Set to container A runtime b) Allocate removes some CPUs from default CPU Set and use them as additional exclusive CPUS for container B which resizes up c) reconcileState loop continues and applies new CPU Set to container B runtime The CPUs that were allocated in step b) are now assigned to both containers A and B in runtime. 2) It didn't consider temporary conflicts when moving CPUs from one container to another. For example: a) container A uses CPUs: 1, 2; container B uses CPUs: 3, 4 b) container B scales down by one cpu, and now only CPU: 3 is allocated for it c) container A scales up and receives CPU: 4 during allocation (so now it has CPUs: 1, 2, 4 allocated) d) reconcileState applies new CPU Set to container A runtime: 1, 2, 4 e) reconcileState applies new CPU Set to container B runtime: 3 Between steps d) and e) CPU: 4 is assigned to both container A and container B. If kubelet is restarted that time, the situation will hold for some time. The new algorithm: 1) Modifies the loop iterating over all containers in all pods to act in a critical section controlled by CPU manager's lock - same that is used during Allocate. During the iteration CPU Sets are not yet applied but only collected to local variables: exclusiveCPUContainers and nonExclusiveCPUContainers. Usage of the lock guarantees consistent state. 2) After collection and outside critical section CPU Sets are applied to runtime in three steps: 2.1) remove scaled down exclusive CPUs from containers * as containers using exclusive CPUs cannot be scaled down to 0, because they need to retain Original CPU Set, it is safe operation that won't try to set an empty set in runtime * after this operation all CPUs that belong now to default CPU Set are no longer used exclusively by any container, so next step can be applied 2.2) apply CPU Sets for all non-exclusive containers * these containers will use default CPUSet * if default CPUSet shrank since last reconcileState call due to allocation of the CPUs as exclusive, the CPUs removed from it are now no longer used by non-exclusive containers, so next step can be applied 2.3) set final CPU Sets for containers using exclusive CPUs 3) Improves conflict detection by tracking failed container updates and preventing further updates that would conflict with previously failed ones. Signed-off-by: Lukasz Wojciechowski --- pkg/kubelet/cm/cpumanager/cpu_manager.go | 98 +++++++++++++++++++----- 1 file changed, 80 insertions(+), 18 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go index 7db6c5223217b..d22a58170ab9d 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go @@ -381,10 +381,16 @@ func (m *manager) GetAllCPUs() cpuset.CPUSet { type reconciledContainer struct { podName string + podUID string containerName string containerID string } +type reconciledContainerAllocation struct { + reconciledContainer + allocatedSet cpuset.CPUSet +} + func (m *manager) removeStaleState(rootLogger logr.Logger) { // Only once all sources are ready do we attempt to remove any stale state. // This ensures that the call to `m.activePods()` below will succeed with @@ -473,15 +479,19 @@ func (m *manager) reconcileState(ctx context.Context) (success []reconciledConta failure = []reconciledContainer{} rootLogger := klog.FromContext(ctx) - m.removeStaleState(rootLogger) + + exclusiveCPUContainers := []reconciledContainerAllocation{} + nonExclusiveCPUContainers := []reconciledContainerAllocation{} + + m.Lock() for _, pod := range m.activePods() { podLogger := klog.LoggerWithValues(rootLogger, "pod", klog.KObj(pod)) pstatus, ok := m.podStatusProvider.GetPodStatus(pod.UID) if !ok { podLogger.V(5).Info("skipping pod; status not found") - failure = append(failure, reconciledContainer{pod.Name, "", ""}) + failure = append(failure, reconciledContainer{pod.Name, string(pod.UID), "", ""}) continue } @@ -493,25 +503,24 @@ func (m *manager) reconcileState(ctx context.Context) (success []reconciledConta containerID, err := findContainerIDByName(&pstatus, container.Name) if err != nil { logger.V(5).Info("skipping container; ID not found in pod status", "err", err) - failure = append(failure, reconciledContainer{pod.Name, container.Name, ""}) + failure = append(failure, reconciledContainer{pod.Name, string(pod.UID), container.Name, ""}) continue } cstatus, err := findContainerStatusByName(&pstatus, container.Name) if err != nil { logger.V(5).Info("skipping container; container status not found in pod status", "err", err) - failure = append(failure, reconciledContainer{pod.Name, container.Name, ""}) + failure = append(failure, reconciledContainer{pod.Name, string(pod.UID), container.Name, ""}) continue } if cstatus.State.Waiting != nil || (cstatus.State.Waiting == nil && cstatus.State.Running == nil && cstatus.State.Terminated == nil) { logger.V(4).Info("skipping container; container still in the waiting state", "err", err) - failure = append(failure, reconciledContainer{pod.Name, container.Name, ""}) + failure = append(failure, reconciledContainer{pod.Name, string(pod.UID), container.Name, ""}) continue } - m.Lock() if cstatus.State.Terminated != nil { // The container is terminated but we can't call m.RemoveContainer() // here because it could remove the allocated cpuset for the container @@ -522,7 +531,6 @@ func (m *manager) reconcileState(ctx context.Context) (success []reconciledConta if err == nil { logger.V(4).Info("ignoring terminated container", "containerID", containerID) } - m.Unlock() continue } @@ -530,30 +538,84 @@ func (m *manager) reconcileState(ctx context.Context) (success []reconciledConta // Idempotently add it to the containerMap incase it is missing. // This can happen after a kubelet restart, for example. m.containerMap.Add(string(pod.UID), container.Name, containerID) - m.Unlock() - cset := m.state.GetCPUSetOrDefault(string(pod.UID), container.Name) + cset, exclusive := m.state.GetCPUSet(string(pod.UID), container.Name) + if !exclusive { + cset = m.state.GetDefaultCPUSet() + } if cset.IsEmpty() { // NOTE: This should not happen outside of tests. logger.V(2).Info("ReconcileState: skipping container; empty cpuset assigned") - failure = append(failure, reconciledContainer{pod.Name, container.Name, containerID}) + failure = append(failure, reconciledContainer{pod.Name, string(pod.UID), container.Name, containerID}) continue } - lcset := m.lastUpdateState.GetCPUSetOrDefault(string(pod.UID), container.Name) - if !cset.Equals(lcset) { - logger.V(5).Info("updating container", "containerID", containerID, "cpuSet", cset) - err = m.updateContainerCPUSet(ctx, containerID, cset) + rca := reconciledContainerAllocation{ + reconciledContainer{pod.Name, string(pod.UID), container.Name, containerID}, + cset, + } + if exclusive { + exclusiveCPUContainers = append(exclusiveCPUContainers, rca) + } else { + nonExclusiveCPUContainers = append(nonExclusiveCPUContainers, rca) + } + + } + } + m.Unlock() + + failedContainersCPUSet := cpuset.New() + + updateContainers := func(containers []reconciledContainerAllocation, preliminary bool) { + for _, rca := range containers { + logger := klog.LoggerWithValues(rootLogger, "podName", rca.podName, "containerName", rca.containerName) + + lcset := m.lastUpdateState.GetCPUSetOrDefault(rca.podUID, rca.containerName) + + // Determine the CPU set to use based on the pass + var targetCPUSet cpuset.CPUSet + if preliminary { + targetCPUSet = rca.allocatedSet.Intersection(lcset) + } else { + targetCPUSet = rca.allocatedSet + } + + // Check if update is needed + if !targetCPUSet.Equals(lcset) { + if !preliminary && !targetCPUSet.Intersection(failedContainersCPUSet).IsEmpty() { + logger.Error(fmt.Errorf("Conflict with previously failed container CPUSet updates"), "failed to update container", "containerID", rca.containerID, "cpuSet", rca.allocatedSet) + failure = append(failure, rca.reconciledContainer) + failedContainersCPUSet = failedContainersCPUSet.Union(lcset) + continue + } + + logger.V(5).Info("updating container", "containerID", rca.containerID, "cpuSet", targetCPUSet) + err := m.updateContainerCPUSet(ctx, rca.containerID, targetCPUSet) if err != nil { - logger.Error(err, "failed to update container", "containerID", containerID, "cpuSet", cset) - failure = append(failure, reconciledContainer{pod.Name, container.Name, containerID}) + logger.Error(err, "failed to update container", "containerID", rca.containerID, "cpuSet", targetCPUSet) + failure = append(failure, rca.reconciledContainer) + failedContainersCPUSet = failedContainersCPUSet.Union(lcset) continue } - m.lastUpdateState.SetCPUSet(string(pod.UID), container.Name, cset) + m.lastUpdateState.SetCPUSet(rca.podUID, rca.containerName, targetCPUSet) + } + + // Add to success list if required + if !preliminary { + success = append(success, rca.reconciledContainer) } - success = append(success, reconciledContainer{pod.Name, container.Name, containerID}) } } + + // first pass - only remove CPUs from containers using exclusive CPUs + updateContainers(exclusiveCPUContainers, true) + + // second pass - apply CPU sets to non exclusive CPU containers + updateContainers(nonExclusiveCPUContainers, false) + + // third pass - apply final CPU set to containers using exclusive CPUs + updateContainers(exclusiveCPUContainers, false) + return success, failure } From 14d7ba1c809f9275a300890e6511d64600858a54 Mon Sep 17 00:00:00 2001 From: Lukasz Wojciechowski Date: Mon, 27 Apr 2026 16:19:51 +0200 Subject: [PATCH 6/6] kubelet: test cpu-manager multi-pass reconcilation This commit adds extensive test coverage for the new multi-pass reconciliation algorithm in the CPU manager, covering various failure scenarios of UpdateContainerResources function during the reconciliation process to ensure proper handling of CPU conflicts. Testcases are added both to TestReconcileState and TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs tests. Signed-off-by: Lukasz Wojciechowski --- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 2253 +++++++++++++---- 1 file changed, 1782 insertions(+), 471 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index 2a60fb34c675d..78aeffb76ec8a 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -1600,8 +1600,663 @@ func TestReconcileState(t *testing.T) { expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, expectFailedContainerName: []string{}, }, + { + description: "cpu manager reconcile - fail in first reconcile pass does not cause conflict", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + nil, //fakeContainerBID pass 2 ok + nil, //fakeContainerCID pass 3 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(5, 8), + }, + }, + stDefaultCPUSet: cpuset.New(4, 7), + lastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(6, 7, 8), + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(5, 8), + }, + }, + expectStDefaultCPUSet: cpuset.New(4, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(4, 7), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(5, 8), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(4, 7), + "fakeContainerCID": cpuset.New(5, 8), + }, + expectSucceededContainerName: []string{"fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{"fakeContainerAName"}, + }, + { + description: "cpu manager reconcile - fail in first reconcile pass causes conflict in second pass", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + nil, //fakeContainerCID pass 3 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(8), + }, + }, + stDefaultCPUSet: cpuset.New(1, 4, 7), + lastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(6, 7, 8), + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(8), + }, + }, + expectStDefaultCPUSet: cpuset.New(1, 4, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(8), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(8), + }, + expectSucceededContainerName: []string{"fakeContainerCName"}, + expectFailedContainerName: []string{"fakeContainerAName", "fakeContainerBName"}, + }, + { + description: "cpu manager reconcile - fail in first reconcile pass causes conflict in second pass which causes conflict in third pass", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(5, 8), + }, + }, + stDefaultCPUSet: cpuset.New(1, 4, 7), + lastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(6, 7, 8), + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(5, 8), + }, + }, + expectStDefaultCPUSet: cpuset.New(1, 4, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(8), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(8), + }, + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + }, + { + description: "cpu manager reconcile - fail in first reconcile pass causes conflict in third pass", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + nil, //fakeContainerBID pass 2 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(2, 5, 8), + }, + }, + stDefaultCPUSet: cpuset.New(4, 7), + lastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(6, 7, 8), + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 3, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(2, 5, 8), + }, + }, + expectStDefaultCPUSet: cpuset.New(4, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(4, 7), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(8), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(4, 7), + "fakeContainerCID": cpuset.New(8), + }, + expectSucceededContainerName: []string{"fakeContainerBName"}, + expectFailedContainerName: []string{"fakeContainerAName", "fakeContainerCName"}, + }, + { + description: "cpu manager reconcile - fail in second reconcile pass causes conflict in third pass", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: []error{ + nil, //fakeContainerAID pass 1 ok + nil, //fakeContainerCID pass 1 ok + fmt.Errorf("fakeContainerBID pass 2 error"), + nil, //fakeContainerAID pass 3 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(2, 5, 8), + }, + }, + stDefaultCPUSet: cpuset.New(1, 4, 7), + lastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 1, 2), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(6, 7, 8), + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 6), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(2, 5, 8), + }, + }, + expectStDefaultCPUSet: cpuset.New(1, 4, 7), + expectLastUpdateStAssignments: state.ContainerCPUAssignments{ + "fakePodAUID": map[string]cpuset.CPUSet{ + "fakeContainerAName": cpuset.New(0, 6), + }, + "fakePodBUID": map[string]cpuset.CPUSet{ + "fakeContainerBName": cpuset.New(3, 4, 5), + }, + "fakePodCUID": map[string]cpuset.CPUSet{ + "fakeContainerCName": cpuset.New(8), + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 6), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(8), + }, + expectSucceededContainerName: []string{"fakeContainerAName"}, + expectFailedContainerName: []string{"fakeContainerBName", "fakeContainerCName"}, + }, } - for _, testCase := range testCases { logger, _ := ktesting.NewTestContext(t) mgr := &manager{ @@ -2273,208 +2928,571 @@ func TestCPUManagerGenerateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing expectedError error }{ { - description: "set none policy", - cpuPolicyName: "none", - nodeAllocatableReservation: nil, - expectedPolicy: "none", + description: "set none policy", + cpuPolicyName: "none", + nodeAllocatableReservation: nil, + expectedPolicy: "none", + }, + { + description: "invalid policy name", + cpuPolicyName: "invalid", + nodeAllocatableReservation: nil, + expectedError: fmt.Errorf("unknown policy: \"invalid\""), + }, + { + description: "static policy", + cpuPolicyName: "static", + nodeAllocatableReservation: v1.ResourceList{v1.ResourceCPU: *resource.NewQuantity(3, resource.DecimalSI)}, + expectedPolicy: "static", + }, + { + description: "static policy - broken topology", + cpuPolicyName: "static", + nodeAllocatableReservation: v1.ResourceList{}, + isTopologyBroken: true, + expectedError: fmt.Errorf("could not detect number of cpus"), + }, + { + description: "static policy - broken reservation", + cpuPolicyName: "static", + nodeAllocatableReservation: v1.ResourceList{}, + expectedError: fmt.Errorf("unable to determine reserved CPU resources for static policy"), + }, + { + description: "static policy - no CPU resources", + cpuPolicyName: "static", + nodeAllocatableReservation: v1.ResourceList{v1.ResourceCPU: *resource.NewQuantity(0, resource.DecimalSI)}, + expectedError: fmt.Errorf("the static policy requires systemreserved.cpu + kubereserved.cpu to be greater than zero"), + }, + } + + mockedMachineInfo := cadvisorapi.MachineInfo{ + NumCores: 4, + Topology: []cadvisorapi.Node{ + { + Cores: []cadvisorapi.Core{ + { + Id: 0, + Threads: []int{0}, + UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + }, + { + Id: 1, + Threads: []int{1}, + UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + }, + { + Id: 2, + Threads: []int{2}, + UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + }, + { + Id: 3, + Threads: []int{3}, + UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + machineInfo := &mockedMachineInfo + if testCase.isTopologyBroken { + machineInfo = &cadvisorapi.MachineInfo{} + } + sDir, err := os.MkdirTemp("", "cpu_manager_test") + if err != nil { + t.Errorf("cannot create state file: %s", err.Error()) + } + defer removeStateDirectory(t, sDir) + + logger, _ := ktesting.NewTestContext(t) + mgr, err := NewManager(logger, testCase.cpuPolicyName, nil, 5*time.Second, machineInfo, cpuset.New(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager()) + if testCase.expectedError != nil { + if !strings.Contains(err.Error(), testCase.expectedError.Error()) { + t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error()) + } + } else { + rawMgr := mgr.(*manager) + if rawMgr.policy.Name() != testCase.expectedPolicy { + t.Errorf("Unexpected policy name. Have: %q wants %q", rawMgr.policy.Name(), testCase.expectedPolicy) + } + if rawMgr.topology == nil { + t.Errorf("Expected topology to be non-nil for policy '%v'. Have: %v", rawMgr.policy.Name(), rawMgr.topology) + } + } + }) + + } +} + +func TestCPUManagerRemoveWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) { + if runtime.GOOS == "windows" { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsCPUAndMemoryAffinity, true) + } + + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScalingExclusiveCPUs, true) + containerID := "fakeID" + containerMap := containermap.NewContainerMap() + + logger, _ := ktesting.NewTestContext(t) + mgr := &manager{ + policy: &mockPolicy{ + err: nil, + }, + state: &mockState{ + allocations: state.ContainerCPUAllocations{}, + defaultCPUSet: cpuset.New(), + }, + lastUpdateState: state.NewMemoryState(logger), + containerRuntime: &mockRuntimeService{}, + containerMap: containerMap, + activePods: func() []*v1.Pod { return nil }, + podStatusProvider: mockPodStatusProvider{}, + } + + containerMap.Add("", "", containerID) + err := mgr.RemoveContainer(logger, containerID) + if err != nil { + t.Errorf("CPU Manager RemoveContainer() error. expected error to be nil but got: %v", err) + } + + mgr = &manager{ + policy: &mockPolicy{ + err: fmt.Errorf("fake error"), + }, + state: state.NewMemoryState(logger), + containerRuntime: &mockRuntimeService{}, + containerMap: containerMap, + activePods: func() []*v1.Pod { return nil }, + podStatusProvider: mockPodStatusProvider{}, + } + + containerMap.Add("", "", containerID) + err = mgr.RemoveContainer(logger, containerID) + if !reflect.DeepEqual(err, fmt.Errorf("fake error")) { + t.Errorf("CPU Manager RemoveContainer() error. expected error: fake error but got: %v", err) + } +} + +func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) { + if runtime.GOOS == "windows" { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsCPUAndMemoryAffinity, true) + } + + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScalingExclusiveCPUs, true) + logger, _ := ktesting.NewTestContext(t) + + testPolicy, _ := NewStaticPolicy( + logger, + &topology.CPUTopology{ + NumCPUs: 8, + NumSockets: 2, + NumCores: 4, + CPUDetails: map[int]topology.CPUInfo{ + 0: {CoreID: 0, SocketID: 0}, + 1: {CoreID: 1, SocketID: 0}, + 2: {CoreID: 2, SocketID: 0}, + 3: {CoreID: 3, SocketID: 0}, + 4: {CoreID: 0, SocketID: 1}, + 5: {CoreID: 1, SocketID: 1}, + 6: {CoreID: 2, SocketID: 1}, + 7: {CoreID: 3, SocketID: 1}, + }, + }, + 0, + cpuset.New(), + topologymanager.NewFakeManager(), + nil) + + testCases := []struct { + description string + policy Policy + activePods []*v1.Pod + pspPS v1.PodStatus + pspFound bool + updateErr []error + containerIDsWithExclusiveCPUs []string + containerRuntimeInitialState map[string]cpuset.CPUSet + stAllocations state.ContainerCPUAllocations + stDefaultCPUSet cpuset.CPUSet + lastUpdateStAllocations state.ContainerCPUAllocations + lastUpdateStDefaultCPUSet cpuset.CPUSet + expectStAllocations state.ContainerCPUAllocations + expectStDefaultCPUSet cpuset.CPUSet + expectLastUpdateStAllocations state.ContainerCPUAllocations + expectLastUpdateStDefaultCPUSet cpuset.CPUSet + expectContainerRuntimeState map[string]cpuset.CPUSet + expectSucceededContainerName []string + expectFailedContainerName []string + }{ + { + description: "cpu manager reconcile - no error", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodName", + UID: "fakePodUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerName", + ContainerID: "docker://fakeContainerID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { - description: "invalid policy name", - cpuPolicyName: "invalid", - nodeAllocatableReservation: nil, - expectedError: fmt.Errorf("unknown policy: \"invalid\""), + description: "cpu manager reconcile init container - no error", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodName", + UID: "fakePodUID", + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "fakeContainerName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + InitContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerName", + ContainerID: "docker://fakeContainerID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, { - description: "static policy", - cpuPolicyName: "static", - nodeAllocatableReservation: v1.ResourceList{v1.ResourceCPU: *resource.NewQuantity(3, resource.DecimalSI)}, - expectedPolicy: "static", + description: "cpu manager reconcile - pod status not found", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodName", + UID: "fakePodUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{}, + pspFound: false, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{}, + expectStDefaultCPUSet: cpuset.New(), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{}, }, { - description: "static policy - broken topology", - cpuPolicyName: "static", - nodeAllocatableReservation: v1.ResourceList{}, - isTopologyBroken: true, - expectedError: fmt.Errorf("could not detect number of cpus"), + description: "cpu manager reconcile - container state not found", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodName", + UID: "fakePodUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerName1", + ContainerID: "docker://fakeContainerID", + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{}, + expectStDefaultCPUSet: cpuset.New(), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{}, }, { - description: "static policy - broken reservation", - cpuPolicyName: "static", - nodeAllocatableReservation: v1.ResourceList{}, - expectedError: fmt.Errorf("unable to determine reserved CPU resources for static policy"), + description: "cpu manager reconclie - cpuset is empty", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodName", + UID: "fakePodUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "fakeContainerName", + ContainerID: "docker://fakeContainerID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(), Resized: cpuset.New()}, + }, + }, + stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(), Resized: cpuset.New()}, + }, + }, + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerName"}, }, { - description: "static policy - no CPU resources", - cpuPolicyName: "static", - nodeAllocatableReservation: v1.ResourceList{v1.ResourceCPU: *resource.NewQuantity(0, resource.DecimalSI)}, - expectedError: fmt.Errorf("the static policy requires systemreserved.cpu + kubereserved.cpu to be greater than zero"), - }, - } - - mockedMachineInfo := cadvisorapi.MachineInfo{ - NumCores: 4, - Topology: []cadvisorapi.Node{ - { - Cores: []cadvisorapi.Core{ - { - Id: 0, - Threads: []int{0}, - UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + description: "cpu manager reconclie - container update error", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodName", + UID: "fakePodUID", }, - { - Id: 1, - Threads: []int{1}, - UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerName", + }, + }, }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ { - Id: 2, - Threads: []int{2}, - UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + Name: "fakeContainerName", + ContainerID: "docker://fakeContainerID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + pspFound: true, + updateErr: []error{fmt.Errorf("fake container update error")}, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{}, + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerName"}, + }, + { + description: "cpu manager reconcile - state has inactive container", + policy: testPolicy, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodName", + UID: "fakePodUID", }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerName", + }, + }, + }, + }, + }, + pspPS: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ { - Id: 3, - Threads: []int{3}, - UncoreCaches: []cadvisorapi.Cache{{Id: 1}}, + Name: "fakeContainerName", + ContainerID: "docker://fakeContainerID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, }, }, }, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.description, func(t *testing.T) { - machineInfo := &mockedMachineInfo - if testCase.isTopologyBroken { - machineInfo = &cadvisorapi.MachineInfo{} - } - sDir, err := os.MkdirTemp("", "cpu_manager_test") - if err != nil { - t.Errorf("cannot create state file: %s", err.Error()) - } - defer removeStateDirectory(t, sDir) - - logger, _ := ktesting.NewTestContext(t) - mgr, err := NewManager(logger, testCase.cpuPolicyName, nil, 5*time.Second, machineInfo, cpuset.New(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager()) - if testCase.expectedError != nil { - if !strings.Contains(err.Error(), testCase.expectedError.Error()) { - t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error()) - } - } else { - rawMgr := mgr.(*manager) - if rawMgr.policy.Name() != testCase.expectedPolicy { - t.Errorf("Unexpected policy name. Have: %q wants %q", rawMgr.policy.Name(), testCase.expectedPolicy) - } - if rawMgr.topology == nil { - t.Errorf("Expected topology to be non-nil for policy '%v'. Have: %v", rawMgr.policy.Name(), rawMgr.topology) - } - } - }) - - } -} - -func TestCPUManagerRemoveWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) { - if runtime.GOOS == "windows" { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsCPUAndMemoryAffinity, true) - } - - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScalingExclusiveCPUs, true) - containerID := "fakeID" - containerMap := containermap.NewContainerMap() - - logger, _ := ktesting.NewTestContext(t) - mgr := &manager{ - policy: &mockPolicy{ - err: nil, - }, - state: &mockState{ - allocations: state.ContainerCPUAllocations{}, - defaultCPUSet: cpuset.New(), - }, - lastUpdateState: state.NewMemoryState(logger), - containerRuntime: &mockRuntimeService{}, - containerMap: containerMap, - activePods: func() []*v1.Pod { return nil }, - podStatusProvider: mockPodStatusProvider{}, - } - - containerMap.Add("", "", containerID) - err := mgr.RemoveContainer(logger, containerID) - if err != nil { - t.Errorf("CPU Manager RemoveContainer() error. expected error to be nil but got: %v", err) - } - - mgr = &manager{ - policy: &mockPolicy{ - err: fmt.Errorf("fake error"), - }, - state: state.NewMemoryState(logger), - containerRuntime: &mockRuntimeService{}, - containerMap: containerMap, - activePods: func() []*v1.Pod { return nil }, - podStatusProvider: mockPodStatusProvider{}, - } - - containerMap.Add("", "", containerID) - err = mgr.RemoveContainer(logger, containerID) - if !reflect.DeepEqual(err, fmt.Errorf("fake error")) { - t.Errorf("CPU Manager RemoveContainer() error. expected error: fake error but got: %v", err) - } -} - -func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) { - if runtime.GOOS == "windows" { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsCPUAndMemoryAffinity, true) - } - - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScalingExclusiveCPUs, true) - logger, _ := ktesting.NewTestContext(t) - - testPolicy, _ := NewStaticPolicy( - logger, - &topology.CPUTopology{ - NumCPUs: 8, - NumSockets: 2, - NumCores: 4, - CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 0}, - 4: {CoreID: 0, SocketID: 1}, - 5: {CoreID: 1, SocketID: 1}, - 6: {CoreID: 2, SocketID: 1}, - 7: {CoreID: 3, SocketID: 1}, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + "secondfakePodUID": map[string]state.ContainerCPUAllocation{ + "secondfakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, + }, + }, + stDefaultCPUSet: cpuset.New(5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, + expectLastUpdateStDefaultCPUSet: cpuset.New(), + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), }, + expectSucceededContainerName: []string{"fakeContainerName"}, + expectFailedContainerName: []string{}, }, - 0, - cpuset.New(), - topologymanager.NewFakeManager(), - nil) - - testCases := []struct { - description string - policy Policy - activePods []*v1.Pod - pspPS v1.PodStatus - pspFound bool - updateErr []error - containerIDsWithExclusiveCPUs []string - containerRuntimeInitialState map[string]cpuset.CPUSet - stAllocations state.ContainerCPUAllocations - stDefaultCPUSet cpuset.CPUSet - lastUpdateStAllocations state.ContainerCPUAllocations - lastUpdateStDefaultCPUSet cpuset.CPUSet - expectStAllocations state.ContainerCPUAllocations - expectStDefaultCPUSet cpuset.CPUSet - expectLastUpdateStAllocations state.ContainerCPUAllocations - expectLastUpdateStDefaultCPUSet cpuset.CPUSet - expectContainerRuntimeState map[string]cpuset.CPUSet - expectSucceededContainerName []string - expectFailedContainerName []string - }{ { - description: "cpu manager reconcile - no error", + description: "cpu manager reconcile - last update state is current", policy: testPolicy, activePods: []*v1.Pod{ { @@ -2505,21 +3523,27 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspFound: true, updateErr: nil, containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(1, 2), + }, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + }, + }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectStDefaultCPUSet: cpuset.New(5, 6, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, @@ -2533,7 +3557,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectFailedContainerName: []string{}, }, { - description: "cpu manager reconcile init container - no error", + description: "cpu manager reconcile - last update state is not current", policy: testPolicy, activePods: []*v1.Pod{ { @@ -2542,7 +3566,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) UID: "fakePodUID", }, Spec: v1.PodSpec{ - InitContainers: []v1.Container{ + Containers: []v1.Container{ { Name: "fakeContainerName", }, @@ -2551,7 +3575,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, pspPS: v1.PodStatus{ - InitContainerStatuses: []v1.ContainerStatus{ + ContainerStatuses: []v1.ContainerStatus{ { Name: "fakeContainerName", ContainerID: "docker://fakeContainerID", @@ -2564,14 +3588,20 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspFound: true, updateErr: nil, containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerID": cpuset.New(3, 4), + }, stAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, }, }, - stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, + }, + }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ @@ -2581,7 +3611,7 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New(1, 2)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), @@ -2592,53 +3622,44 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) expectFailedContainerName: []string{}, }, { - description: "cpu manager reconcile - pod status not found", + description: "cpu manager reconcile - default CPU sets no error", policy: testPolicy, activePods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ - Name: "fakePodName", - UID: "fakePodUID", + Name: "fakePodAName", + UID: "fakePodAUID", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "fakeContainerName", + Name: "fakeContainerAName", }, }, }, }, - }, - pspPS: v1.PodStatus{}, - pspFound: false, - updateErr: nil, - containerIDsWithExclusiveCPUs: []string{}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, - stAllocations: state.ContainerCPUAllocations{}, - stDefaultCPUSet: cpuset.New(), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAllocations: state.ContainerCPUAllocations{}, - expectStDefaultCPUSet: cpuset.New(), - expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, - expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectContainerRuntimeState: map[string]cpuset.CPUSet{}, - expectSucceededContainerName: []string{}, - expectFailedContainerName: []string{}, - }, - { - description: "cpu manager reconcile - container state not found", - policy: testPolicy, - activePods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ - Name: "fakePodName", - UID: "fakePodUID", + Name: "fakePodBName", + UID: "fakePodBUID", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "fakeContainerName", + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", }, }, }, @@ -2647,40 +3668,97 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspPS: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { - Name: "fakeContainerName1", - ContainerID: "docker://fakeContainerID", + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, }, }, }, - pspFound: true, - updateErr: nil, - containerIDsWithExclusiveCPUs: []string{}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, - stAllocations: state.ContainerCPUAllocations{}, - stDefaultCPUSet: cpuset.New(), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAllocations: state.ContainerCPUAllocations{}, - expectStDefaultCPUSet: cpuset.New(), - expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, + pspFound: true, + updateErr: nil, + containerIDsWithExclusiveCPUs: []string{}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + stAllocations: state.ContainerCPUAllocations{}, + stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{}, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{}, + expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + }, + }, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectContainerRuntimeState: map[string]cpuset.CPUSet{}, - expectSucceededContainerName: []string{}, - expectFailedContainerName: []string{}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + "fakeContainerBID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + "fakeContainerCID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, }, { - description: "cpu manager reconclie - cpuset is empty", + description: "cpu manager reconcile - exclusive cpu container scaled up", policy: testPolicy, activePods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ - Name: "fakePodName", - UID: "fakePodUID", + Name: "fakePodAName", + UID: "fakePodAUID", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "fakeContainerName", + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", }, }, }, @@ -2689,8 +3767,22 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspPS: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { - Name: "fakeContainerName", - ContainerID: "docker://fakeContainerID", + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", State: v1.ContainerState{ Running: &v1.ContainerStateRunning{}, }, @@ -2699,94 +3791,95 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, pspFound: true, updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + containerIDsWithExclusiveCPUs: []string{"fakeContainerBID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 5, 6, 7), + "fakeContainerBID": cpuset.New(3, 4), + "fakeContainerCID": cpuset.New(1, 2, 5, 6, 7), + }, stAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(), Resized: cpuset.New()}, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + }, + }, + stDefaultCPUSet: cpuset.New(1, 2, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New()}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New()}, }, }, - stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(), Resized: cpuset.New()}, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + }, + }, + expectStDefaultCPUSet: cpuset.New(1, 2, 7), + expectLastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New(1, 2, 7)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New(1, 2, 7)}, }, }, - expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), - expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectContainerRuntimeState: map[string]cpuset.CPUSet{}, - expectSucceededContainerName: []string{}, - expectFailedContainerName: []string{"fakeContainerName"}, + expectContainerRuntimeState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 7), + "fakeContainerBID": cpuset.New(3, 4, 5, 6), + "fakeContainerCID": cpuset.New(1, 2, 7), + }, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{}, }, { - description: "cpu manager reconclie - container update error", + description: "cpu manager reconcile - exclusive cpu container scaled down", policy: testPolicy, activePods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ - Name: "fakePodName", - UID: "fakePodUID", + Name: "fakePodAName", + UID: "fakePodAUID", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "fakeContainerName", + Name: "fakeContainerAName", }, }, }, }, - }, - pspPS: v1.PodStatus{ - ContainerStatuses: []v1.ContainerStatus{ - { - Name: "fakeContainerName", - ContainerID: "docker://fakeContainerID", - State: v1.ContainerState{ - Running: &v1.ContainerStateRunning{}, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, }, }, }, - }, - pspFound: true, - updateErr: []error{fmt.Errorf("fake container update error")}, - containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, - stAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, - }, - }, - stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, - }, - }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), - expectLastUpdateStAllocations: state.ContainerCPUAllocations{}, - expectLastUpdateStDefaultCPUSet: cpuset.New(), - expectContainerRuntimeState: map[string]cpuset.CPUSet{}, - expectSucceededContainerName: []string{}, - expectFailedContainerName: []string{"fakeContainerName"}, - }, - { - description: "cpu manager reconcile - state has inactive container", - policy: testPolicy, - activePods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ - Name: "fakePodName", - UID: "fakePodUID", + Name: "fakePodCName", + UID: "fakePodCUID", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "fakeContainerName", + Name: "fakeContainerCName", }, }, }, @@ -2795,8 +3888,22 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspPS: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { - Name: "fakeContainerName", - ContainerID: "docker://fakeContainerID", + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", State: v1.ContainerState{ Running: &v1.ContainerStateRunning{}, }, @@ -2805,50 +3912,82 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, pspFound: true, updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, + containerIDsWithExclusiveCPUs: []string{"fakeContainerBID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(1, 2, 7), + "fakeContainerBID": cpuset.New(3, 4, 5, 6), + "fakeContainerCID": cpuset.New(1, 2, 7), + }, stAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4)}, }, - "secondfakePodUID": map[string]state.ContainerCPUAllocation{ - "secondfakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, + }, + stDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New()}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4, 5, 6), Resized: cpuset.New()}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New()}, }, }, - stDefaultCPUSet: cpuset.New(5, 6, 7), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4)}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectStDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New(1, 2, 5, 6, 7)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3, 4, 5, 6), Resized: cpuset.New(3, 4)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New(1, 2, 5, 6, 7)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerID": cpuset.New(1, 2), + "fakeContainerAID": cpuset.New(1, 2, 5, 6, 7), + "fakeContainerBID": cpuset.New(3, 4), + "fakeContainerCID": cpuset.New(1, 2, 5, 6, 7), }, - expectSucceededContainerName: []string{"fakeContainerName"}, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, expectFailedContainerName: []string{}, }, { - description: "cpu manager reconcile - last update state is current", + description: "cpu manager reconcile - exclusive cpu containers swap CPUs", policy: testPolicy, activePods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ - Name: "fakePodName", - UID: "fakePodUID", + Name: "fakePodAName", + UID: "fakePodAUID", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "fakeContainerName", + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", }, }, }, @@ -2857,8 +3996,15 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspPS: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { - Name: "fakeContainerName", - ContainerID: "docker://fakeContainerID", + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", State: v1.ContainerState{ Running: &v1.ContainerStateRunning{}, }, @@ -2867,53 +4013,93 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, pspFound: true, updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerBID"}, containerRuntimeInitialState: map[string]cpuset.CPUSet{ - "fakeContainerID": cpuset.New(1, 2), + "fakeContainerAID": cpuset.New(1, 2), + "fakeContainerBID": cpuset.New(3, 4), }, stAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, }, }, stDefaultCPUSet: cpuset.New(5, 6, 7), lastUpdateStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3, 4)}, }, }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, }, }, expectStDefaultCPUSet: cpuset.New(5, 6, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerID": cpuset.New(1, 2), + "fakeContainerAID": cpuset.New(1, 4), + "fakeContainerBID": cpuset.New(2, 3), }, - expectSucceededContainerName: []string{"fakeContainerName"}, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName"}, expectFailedContainerName: []string{}, }, { - description: "cpu manager reconcile - last update state is not current", + description: "cpu manager reconcile - exclusive cpu containers scaled down and up", policy: testPolicy, activePods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ - Name: "fakePodName", - UID: "fakePodUID", + Name: "fakePodAName", + UID: "fakePodAUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerAName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodBName", + UID: "fakePodBUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerBName", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "fakeContainerName", + Name: "fakeContainerCName", }, }, }, @@ -2922,8 +4108,22 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) pspPS: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { - Name: "fakeContainerName", - ContainerID: "docker://fakeContainerID", + Name: "fakeContainerAName", + ContainerID: "docker://fakeContainerAID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerBName", + ContainerID: "docker://fakeContainerBID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", State: v1.ContainerState{ Running: &v1.ContainerStateRunning{}, }, @@ -2932,42 +4132,64 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, pspFound: true, updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerID"}, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerBID"}, containerRuntimeInitialState: map[string]cpuset.CPUSet{ - "fakeContainerID": cpuset.New(3, 4), + "fakeContainerAID": cpuset.New(1, 2), + "fakeContainerBID": cpuset.New(3, 4), + "fakeContainerCID": cpuset.New(5, 6), }, stAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, }, }, - stDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + stDefaultCPUSet: cpuset.New(4), lastUpdateStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3, 4)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(5, 6), Resized: cpuset.New()}, }, }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(1, 2), Resized: cpuset.New()}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, }, }, - expectStDefaultCPUSet: cpuset.New(3, 4, 5, 6, 7), + expectStDefaultCPUSet: cpuset.New(4), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ - "fakePodUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerName": {Original: cpuset.New(3, 4), Resized: cpuset.New(1, 2)}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(5, 6), Resized: cpuset.New(4)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerID": cpuset.New(1, 2), + "fakeContainerAID": cpuset.New(1, 2, 5, 6), + "fakeContainerBID": cpuset.New(3), + "fakeContainerCID": cpuset.New(4), }, - expectSucceededContainerName: []string{"fakeContainerName"}, + expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, expectFailedContainerName: []string{}, }, { - description: "cpu manager reconcile - default CPU sets no error", + description: "cpu manager reconcile - fail in first reconcile pass does not cause conflict", policy: testPolicy, activePods: []*v1.Pod{ { @@ -3035,38 +4257,71 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, - containerIDsWithExclusiveCPUs: []string{}, - containerRuntimeInitialState: map[string]cpuset.CPUSet{}, - stAllocations: state.ContainerCPUAllocations{}, - stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), - lastUpdateStAllocations: state.ContainerCPUAllocations{}, - lastUpdateStDefaultCPUSet: cpuset.New(), - expectStAllocations: state.ContainerCPUAllocations{}, - expectStDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + nil, //fakeContainerBID pass 2 ok + nil, //fakeContainerCID pass 3 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, + containerRuntimeInitialState: map[string]cpuset.CPUSet{ + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), + }, + stAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(5, 8)}, + }, + }, + stDefaultCPUSet: cpuset.New(4, 7), + lastUpdateStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, + }, + "fakePodBUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(6, 7, 8)}, + }, + }, + lastUpdateStDefaultCPUSet: cpuset.New(), + expectStAllocations: state.ContainerCPUAllocations{ + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(5, 8)}, + }, + }, + expectStDefaultCPUSet: cpuset.New(4, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(4, 7)}, }, "fakePodCUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerCName": {Original: cpuset.New(1, 2, 3, 4, 5, 6, 7), Resized: cpuset.New()}, + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(5, 8)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2, 3, 4, 5, 6, 7), - "fakeContainerBID": cpuset.New(1, 2, 3, 4, 5, 6, 7), - "fakeContainerCID": cpuset.New(1, 2, 3, 4, 5, 6, 7), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(4, 7), + "fakeContainerCID": cpuset.New(5, 8), }, - expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, - expectFailedContainerName: []string{}, + expectSucceededContainerName: []string{"fakeContainerBName", "fakeContainerCName"}, + expectFailedContainerName: []string{"fakeContainerAName"}, }, { - description: "cpu manager reconcile - exclusive cpu container scaled up", + description: "cpu manager reconcile - fail in first reconcile pass causes conflict in second pass", policy: testPolicy, activePods: []*v1.Pod{ { @@ -3134,60 +4389,70 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerBID"}, + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + nil, //fakeContainerCID pass 3 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, containerRuntimeInitialState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2, 5, 6, 7), - "fakeContainerBID": cpuset.New(3, 4), - "fakeContainerCID": cpuset.New(1, 2, 5, 6, 7), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), }, stAllocations: state.ContainerCPUAllocations{ - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(8)}, }, }, - stDefaultCPUSet: cpuset.New(1, 2, 7), + stDefaultCPUSet: cpuset.New(1, 4, 7), lastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New()}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New()}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, }, "fakePodCUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerCName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New()}, + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(6, 7, 8)}, }, }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(8)}, }, }, - expectStDefaultCPUSet: cpuset.New(1, 2, 7), + expectStDefaultCPUSet: cpuset.New(1, 4, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New(1, 2, 7)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4, 5, 6)}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, }, "fakePodCUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerCName": {Original: cpuset.New(1, 2, 5, 6, 7), Resized: cpuset.New(1, 2, 7)}, + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(8)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2, 7), - "fakeContainerBID": cpuset.New(3, 4, 5, 6), - "fakeContainerCID": cpuset.New(1, 2, 7), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(8), }, - expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, - expectFailedContainerName: []string{}, + expectSucceededContainerName: []string{"fakeContainerCName"}, + expectFailedContainerName: []string{"fakeContainerAName", "fakeContainerBName"}, }, { - description: "cpu manager reconcile - exclusive cpu container scaled down", + description: "cpu manager reconcile - fail in first reconcile pass causes conflict in second pass which causes conflict in third pass", policy: testPolicy, activePods: []*v1.Pod{ { @@ -3255,60 +4520,69 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerBID"}, + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, containerRuntimeInitialState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2, 7), - "fakeContainerBID": cpuset.New(3, 4, 5, 6), - "fakeContainerCID": cpuset.New(1, 2, 7), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), }, stAllocations: state.ContainerCPUAllocations{ - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4)}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(5, 8)}, }, }, - stDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), + stDefaultCPUSet: cpuset.New(1, 4, 7), lastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New()}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4, 5, 6), Resized: cpuset.New()}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, }, "fakePodCUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerCName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New()}, + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(6, 7, 8)}, }, }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4), Resized: cpuset.New(3, 4)}, + "fakePodAUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(5, 8)}, }, }, - expectStDefaultCPUSet: cpuset.New(1, 2, 5, 6, 7), + expectStDefaultCPUSet: cpuset.New(1, 4, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New(1, 2, 5, 6, 7)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3, 4, 5, 6), Resized: cpuset.New(3, 4)}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, }, "fakePodCUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerCName": {Original: cpuset.New(1, 2, 7), Resized: cpuset.New(1, 2, 5, 6, 7)}, + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(8)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2, 5, 6, 7), - "fakeContainerBID": cpuset.New(3, 4), - "fakeContainerCID": cpuset.New(1, 2, 5, 6, 7), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(8), }, - expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, - expectFailedContainerName: []string{}, + expectSucceededContainerName: []string{}, + expectFailedContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, }, { - description: "cpu manager reconcile - exclusive cpu containers swap CPUs", + description: "cpu manager reconcile - fail in first reconcile pass causes conflict in third pass", policy: testPolicy, activePods: []*v1.Pod{ { @@ -3337,6 +4611,19 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakePodCName", + UID: "fakePodCUID", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainerCName", + }, + }, + }, + }, }, pspPS: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -3354,60 +4641,79 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) Running: &v1.ContainerStateRunning{}, }, }, + { + Name: "fakeContainerCName", + ContainerID: "docker://fakeContainerCID", + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, }, }, - pspFound: true, - updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerBID"}, + pspFound: true, + updateErr: []error{ + fmt.Errorf("fakeContainerAID pass 1 error"), + nil, //fakeContainerCID pass 1 ok + nil, //fakeContainerBID pass 2 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, containerRuntimeInitialState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2), - "fakeContainerBID": cpuset.New(3, 4), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), }, stAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, }, - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(2, 5, 8)}, }, }, - stDefaultCPUSet: cpuset.New(5, 6, 7), + stDefaultCPUSet: cpuset.New(4, 7), lastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3, 4)}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(6, 7, 8)}, }, }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 3, 6)}, }, - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(2, 5, 8)}, }, }, - expectStDefaultCPUSet: cpuset.New(5, 6, 7), + expectStDefaultCPUSet: cpuset.New(4, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 4)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(2, 3)}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(4, 7)}, + }, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(8)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 4), - "fakeContainerBID": cpuset.New(2, 3), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(4, 7), + "fakeContainerCID": cpuset.New(8), }, - expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName"}, - expectFailedContainerName: []string{}, + expectSucceededContainerName: []string{"fakeContainerBName"}, + expectFailedContainerName: []string{"fakeContainerAName", "fakeContainerCName"}, }, { - description: "cpu manager reconcile - exclusive cpu containers scaled down and up", + description: "cpu manager reconcile - fail in second reconcile pass causes conflict in third pass", policy: testPolicy, activePods: []*v1.Pod{ { @@ -3475,63 +4781,68 @@ func TestReconcileStateWithInPlacePodVerticalScalingExclusiveCPUs(t *testing.T) }, }, }, - pspFound: true, - updateErr: nil, - containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerBID"}, + pspFound: true, + updateErr: []error{ + nil, //fakeContainerAID pass 1 ok + nil, //fakeContainerCID pass 1 ok + fmt.Errorf("fakeContainerBID pass 2 error"), + nil, //fakeContainerAID pass 3 ok + }, + containerIDsWithExclusiveCPUs: []string{"fakeContainerAID", "fakeContainerCID"}, containerRuntimeInitialState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2), - "fakeContainerBID": cpuset.New(3, 4), - "fakeContainerCID": cpuset.New(5, 6), + "fakeContainerAID": cpuset.New(0, 1, 2), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(6, 7, 8), }, stAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 6)}, }, - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(2, 5, 8)}, }, }, - stDefaultCPUSet: cpuset.New(4), + stDefaultCPUSet: cpuset.New(1, 4, 7), lastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 1, 2)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3, 4)}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, }, "fakePodCUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerCName": {Original: cpuset.New(5, 6), Resized: cpuset.New()}, + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(6, 7, 8)}, }, }, lastUpdateStDefaultCPUSet: cpuset.New(), expectStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 6)}, }, - "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, + "fakePodCUID": map[string]state.ContainerCPUAllocation{ + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(2, 5, 8)}, }, }, - expectStDefaultCPUSet: cpuset.New(4), + expectStDefaultCPUSet: cpuset.New(1, 4, 7), expectLastUpdateStAllocations: state.ContainerCPUAllocations{ "fakePodAUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerAName": {Original: cpuset.New(1), Resized: cpuset.New(1, 2, 5, 6)}, + "fakeContainerAName": {Original: cpuset.New(0), Resized: cpuset.New(0, 6)}, }, "fakePodBUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerBName": {Original: cpuset.New(3), Resized: cpuset.New(3)}, + "fakeContainerBName": {Original: cpuset.New(4), Resized: cpuset.New(3, 4, 5)}, }, "fakePodCUID": map[string]state.ContainerCPUAllocation{ - "fakeContainerCName": {Original: cpuset.New(5, 6), Resized: cpuset.New(4)}, + "fakeContainerCName": {Original: cpuset.New(8), Resized: cpuset.New(8)}, }, }, expectLastUpdateStDefaultCPUSet: cpuset.New(), expectContainerRuntimeState: map[string]cpuset.CPUSet{ - "fakeContainerAID": cpuset.New(1, 2, 5, 6), - "fakeContainerBID": cpuset.New(3), - "fakeContainerCID": cpuset.New(4), + "fakeContainerAID": cpuset.New(0, 6), + "fakeContainerBID": cpuset.New(3, 4, 5), + "fakeContainerCID": cpuset.New(8), }, - expectSucceededContainerName: []string{"fakeContainerAName", "fakeContainerBName", "fakeContainerCName"}, - expectFailedContainerName: []string{}, + expectSucceededContainerName: []string{"fakeContainerAName"}, + expectFailedContainerName: []string{"fakeContainerBName", "fakeContainerCName"}, }, }