From 23d0ceafacdfa3430a34787770bdb52bd109d5b1 Mon Sep 17 00:00:00 2001 From: Timo Zingel Date: Sun, 23 Aug 2026 07:43:15 +0200 Subject: [PATCH] feat: Add BM upgrader for self-managed bare metal clusters MUO already does the pieces a bare-metal upgrade needs: availability and health checks, CVO-driven upgrade, and node keeper/drain. This adds a BM type that reuses that path without OCM or extra-worker scaling, so those clusters can use MUO with a local UpgradeConfig. --- README.md | 21 +++- api/v1alpha1/upgradeconfig_types.go | 4 +- ...e.managed.openshift.io_upgradeconfigs.yaml | 1 + ...econfigs.upgrade.managed.openshift.io.yaml | 1 + ...econfigs.upgrade.managed.openshift.io.yaml | 1 + docs/configmap.md | 8 +- docs/controllers/upgradeconfig.md | 7 ++ docs/design.md | 26 ++--- pkg/specprovider/config.go | 2 +- pkg/upgraders/bmupgrader.go | 99 +++++++++++++++++++ pkg/upgraders/builder.go | 6 ++ 11 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 pkg/upgraders/bmupgrader.go diff --git a/README.md b/README.md index 429ae559..8546887b 100644 --- a/README.md +++ b/README.md @@ -82,4 +82,23 @@ spec: PDBForceDrainTimeout: 60 desired: image: "quay.io/openshift-release-dev/ocp-release@sha256:783a2c963f35ccab38e82e6a8c7fa954c3a4551e07d2f43c06098828dd986ed4" -``` \ No newline at end of file +``` + +### Example 3 - BM upgrade using a local UpgradeConfig + +BM upgrades are scheduled with a local `UpgradeConfig` (`configManager.source: LOCAL`). But extra worker capacity is not reserved. + +```yaml +apiVersion: upgrade.managed.openshift.io/v1alpha1 +kind: UpgradeConfig +metadata: + name: managed-upgrade-config +spec: + type: "BM" + upgradeAt: "2026-08-23T13:12:00Z" + PDBForceDrainTimeout: 60 + capacityReservation: false + desired: + channel: "stable-4.21" + version: "4.21.12" +``` diff --git a/api/v1alpha1/upgradeconfig_types.go b/api/v1alpha1/upgradeconfig_types.go index 323ed769..573ae18c 100644 --- a/api/v1alpha1/upgradeconfig_types.go +++ b/api/v1alpha1/upgradeconfig_types.go @@ -15,6 +15,8 @@ const ( OSD UpgradeType = "OSD" // ARO is a type of upgrade ARO UpgradeType = "ARO" + // BM is a type of upgrade for self-managed bare metal clusters + BM UpgradeType = "BM" ) // FeatureGate type defines the feature that managed-upgrade-operator should enable/disable when deployed. @@ -43,7 +45,7 @@ type UpgradeConfigSpec struct { // The maximum grace period granted to a node whose drain is blocked by a Pod Disruption Budget, before that drain is forced. Measured in minutes. The minimum accepted value is 0 and in this case it will trigger force drain after the expectedNodeDrainTime lapsed. PDBForceDrainTimeout int32 `json:"PDBForceDrainTimeout"` - // +kubebuilder:validation:Enum={"OSD","ARO"} + // +kubebuilder:validation:Enum={"OSD","ARO","BM"} // Type indicates the ClusterUpgrader implementation to use to perform an upgrade of the cluster Type UpgradeType `json:"type"` diff --git a/deploy/crds/upgrade.managed.openshift.io_upgradeconfigs.yaml b/deploy/crds/upgrade.managed.openshift.io_upgradeconfigs.yaml index af314410..58410fec 100644 --- a/deploy/crds/upgrade.managed.openshift.io_upgradeconfigs.yaml +++ b/deploy/crds/upgrade.managed.openshift.io_upgradeconfigs.yaml @@ -93,6 +93,7 @@ spec: enum: - OSD - ARO + - BM type: string upgradeAt: description: Specify the upgrade start time diff --git a/deploy_pko/.test-fixtures/config-with-proxy/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml b/deploy_pko/.test-fixtures/config-with-proxy/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml index 556b5002..c78fec9f 100755 --- a/deploy_pko/.test-fixtures/config-with-proxy/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml +++ b/deploy_pko/.test-fixtures/config-with-proxy/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml @@ -88,6 +88,7 @@ spec: enum: - OSD - ARO + - BM type: string upgradeAt: description: Specify the upgrade start time diff --git a/deploy_pko/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml b/deploy_pko/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml index 556b5002..c78fec9f 100644 --- a/deploy_pko/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml +++ b/deploy_pko/CustomResourceDefinition-upgradeconfigs.upgrade.managed.openshift.io.yaml @@ -88,6 +88,7 @@ spec: enum: - OSD - ARO + - BM type: string upgradeAt: description: Specify the upgrade start time diff --git a/docs/configmap.md b/docs/configmap.md index cc803611..f73b72ef 100644 --- a/docs/configmap.md +++ b/docs/configmap.md @@ -6,6 +6,7 @@ - [How to use it](#how-to-use-it) - [For OSD](#for-osd) - [For ARO](#for-aro) + - [For BM](#for-bm) - [Configurable knobs](#configurable-knobs) - [configManager](#configmanager) - [maintenance](#maintenance) @@ -31,6 +32,10 @@ Maintained in the [managed-cluster-config](https://github.com/openshift/managed- Maintained in the [ARO-RP](https://github.com/Azure/ARO-RP) repository: - https://github.com/Azure/ARO-RP/blob/master/pkg/operator/controllers/muo/staticresources/config.yaml +### For BM cluster + +Maintained in your own individual cluster-config repository, but the BM upgrades are driven from a local `UpgradeConfig` on the cluster. Set `upgradeType: BM` and `configManager.source: LOCAL`. Extra worker scaling is not part of this upgrader, set `capacityReservation: false` on the `UpgradeConfig`. + ## Configurable knobs #### upgradeType @@ -40,6 +45,7 @@ This defines which upgrader MUO should use to upgrade the cluster. Valid options are: - [ARO](https://github.com/openshift/managed-upgrade-operator/blob/master/pkg/upgraders/aroupgrader.go) - [OSD](https://github.com/openshift/managed-upgrade-operator/blob/master/pkg/upgraders/osdupgrader.go) +- [BM](https://github.com/openshift/managed-upgrade-operator/blob/master/pkg/upgraders/bmupgrader.go) If this field is not present or is an empty value, the ARO upgrader is used by default. @@ -109,7 +115,7 @@ Example: #### upgradeWindow -The `upgradeWindow` section is used to control the `managed-upgrade-operator`'s behaviour in relation to the upgrade window within which an upgrade should take place. +The `upgradeWindow` section is used to control the `managed-upgrade-operator`'s behaviour in relation to the upgrade window within which an upgrade should take place. | Key | Description | | --- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| diff --git a/docs/controllers/upgradeconfig.md b/docs/controllers/upgradeconfig.md index d8d6c089..9e4ad31a 100644 --- a/docs/controllers/upgradeconfig.md +++ b/docs/controllers/upgradeconfig.md @@ -195,6 +195,7 @@ MUO has multiple implementations of the upgrade engine which provide their own c - [OSD](../../pkg/upgraders/osdupgrader.go) - [ARO](../../pkg/upgraders/aroupgrader.go) +- [BM](../../pkg/upgraders/bmupgrader.go) MUO [decides](../../pkg/upgraders/builder.go) which upgrader to use based upon the `upgradeType` [configuration](../design.md#configuration). @@ -255,6 +256,12 @@ To integrate your step into the upgrader process, you should: - Add it to the `steps` of whichever upgrader should run it, ie the [OSD upgrader](../../pkg/upgraders/osdupgrader.go), in the specific position order that it should be executed as part of the upgrade process. +### BM Upgrader + +The [BM upgrader](../../pkg/upgraders/bmupgrader.go) follows the ARO sequence without extra-worker scaling. It does not talk to OCM, does not expire upgrades that miss the upgrade window, and does not run FedRAMP post-upgrade procedures. + +It does run `IsUpgradeable` (the CVO `Upgradeable` condition) before health checks, then maintenance windows, `CommenceUpgrade` against ClusterVersion, worker drain/upgrade, and post-upgrade health checks. + ### OSD Upgrader The following flow describes the order and process of the [OSD Upgrader](../../pkg/upgraders/osdupgrader.go). diff --git a/docs/design.md b/docs/design.md index fb2690dc..ab2b3c6a 100644 --- a/docs/design.md +++ b/docs/design.md @@ -28,7 +28,7 @@ For the purpose of upgrading a cluster, an `UpgradeConfig` resource _must_ be co | Item | Definition | Example | | ---- | ---------- | ------- | -| `type` | The cluster upgrader to use when upgrading (valid values: `OSD`, `ARO`)| `OSD` | +| `type` | The cluster upgrader to use when upgrading (valid values: `OSD`, `ARO`, `BM`)| `OSD` | | `upgradeAt` | Timestamp indicating when the upgrade can commence (ISO-8601)| `2020-05-01T12:00:00Z` | | `PDBForceDrainTimeout` | Duration in minutes that a PDB-blocked node is allowed to drain before a drain is forced | `120` | | `desired.version` | The desired OCP release to upgrade to | `4.4.6` | @@ -88,25 +88,25 @@ A fully-populated example of an `UpgradeConfig` status is included below: status: history: - phase: Upgraded - version: 4.3.26 - startTime: "2020-07-05T01:35:36Z" - completeTime: "2020-07-05T03:15:37Z" - conditions: + version: 4.3.26 + startTime: "2020-07-05T01:35:36Z" + completeTime: "2020-07-05T03:15:37Z" + conditions: - completeTime: "2020-07-05T03:15:36Z" lastProbeTime: "2020-07-05T03:15:36Z" - lastTransitionTime: "2020-07-05T03:15:36Z" - message: ScaleUpExtraNodes succeed + lastTransitionTime: "2020-07-05T03:15:36Z" + message: ScaleUpExtraNodes succeed reason: ScaleUpExtraNodes succeed startTime: "2020-07-05T03:15:36Z" - status: "True" + status: "True" type: ScaleUpExtraNodes - completeTime: "2020-07-05T03:15:36Z" lastProbeTime: "2020-07-05T03:15:36Z" - lastTransitionTime: "2020-07-05T03:15:36Z" - message: PreHealthCheck succeed - reason: PreHealthCheck succeed - startTime: "2020-07-05T03:15:36Z" - status: "True" + lastTransitionTime: "2020-07-05T03:15:36Z" + message: PreHealthCheck succeed + reason: PreHealthCheck succeed + startTime: "2020-07-05T03:15:36Z" + status: "True" type: PreHealthCheck ``` diff --git a/pkg/specprovider/config.go b/pkg/specprovider/config.go index a5975244..47b5e26d 100644 --- a/pkg/specprovider/config.go +++ b/pkg/specprovider/config.go @@ -54,7 +54,7 @@ func (cfg *SpecProviderConfig) IsValid() error { } switch upgradev1alpha1.UpgradeType(cfg.UpgradeType) { - case upgradev1alpha1.ARO, upgradev1alpha1.OSD, "": + case upgradev1alpha1.ARO, upgradev1alpha1.OSD, upgradev1alpha1.BM, "": // An empty upgrade type is fine break default: diff --git a/pkg/upgraders/bmupgrader.go b/pkg/upgraders/bmupgrader.go new file mode 100644 index 00000000..5ac3b209 --- /dev/null +++ b/pkg/upgraders/bmupgrader.go @@ -0,0 +1,99 @@ +package upgraders + +import ( + "context" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" + + upgradev1alpha1 "github.com/openshift/managed-upgrade-operator/api/v1alpha1" + ac "github.com/openshift/managed-upgrade-operator/pkg/availabilitychecks" + cv "github.com/openshift/managed-upgrade-operator/pkg/clusterversion" + "github.com/openshift/managed-upgrade-operator/pkg/configmanager" + "github.com/openshift/managed-upgrade-operator/pkg/drain" + "github.com/openshift/managed-upgrade-operator/pkg/eventmanager" + "github.com/openshift/managed-upgrade-operator/pkg/machinery" + "github.com/openshift/managed-upgrade-operator/pkg/maintenance" + "github.com/openshift/managed-upgrade-operator/pkg/metrics" + "github.com/openshift/managed-upgrade-operator/pkg/scaler" + "github.com/openshift/managed-upgrade-operator/pkg/upgradesteps" +) + +// bmUpgrader is a cluster upgrader suitable for self-managed bare metal clusters. +// It inherits from the base clusterUpgrader. +type bmUpgrader struct { + *clusterUpgrader +} + +// NewBMUpgrader creates a new instance of a bmUpgrader +func NewBMUpgrader(c client.Client, cfm configmanager.ConfigManager, mc metrics.Metrics, notifier eventmanager.EventManager) (*bmUpgrader, error) { + cfg := &upgraderConfig{} + err := cfm.Into(cfg) + if err != nil { + return nil, err + } + + m, err := maintenance.NewBuilder().NewClient(c) + if err != nil { + return nil, err + } + + acs, err := ac.GetAvailabilityCheckers(&cfg.ExtDependencyAvailabilityCheck) + if err != nil { + return nil, err + } + + bu := bmUpgrader{ + clusterUpgrader: &clusterUpgrader{ + client: c, + metrics: mc, + cvClient: cv.NewCVClient(c), + notifier: notifier, + config: cfg, + scaler: scaler.NewScaler(), + drainstrategyBuilder: drain.NewBuilder(), + maintenance: m, + machinery: machinery.NewMachinery(), + availabilityCheckers: acs, + }, + } + + bu.steps = bmUpgradeSteps(&bu) + + return &bu, nil +} + +// bmUpgradeSteps returns the ordered upgrade steps for BM clusters. +// Capacity reservation (extra worker scaling) is intentionally omitted. +func bmUpgradeSteps(u *bmUpgrader) []upgradesteps.UpgradeStep { + return []upgradesteps.UpgradeStep{ + upgradesteps.Action(string(upgradev1alpha1.SendStartedNotification), u.SendStartedNotification), + upgradesteps.Action(string(upgradev1alpha1.IsClusterUpgradable), u.IsUpgradeable), + upgradesteps.Action(string(upgradev1alpha1.UpgradePreHealthCheck), u.PreUpgradeHealthCheck), + upgradesteps.Action(string(upgradev1alpha1.ExtDepAvailabilityCheck), u.ExternalDependencyAvailabilityCheck), + upgradesteps.Action(string(upgradev1alpha1.ControlPlaneMaintWindow), u.CreateControlPlaneMaintWindow), + upgradesteps.Action(string(upgradev1alpha1.CommenceUpgrade), u.CommenceUpgrade), + upgradesteps.Action(string(upgradev1alpha1.ControlPlaneUpgraded), u.ControlPlaneUpgraded), + upgradesteps.Action(string(upgradev1alpha1.RemoveControlPlaneMaintWindow), u.RemoveControlPlaneMaintWindow), + upgradesteps.Action(string(upgradev1alpha1.WorkersMaintWindow), u.CreateWorkerMaintWindow), + upgradesteps.Action(string(upgradev1alpha1.AllWorkerNodesUpgraded), u.AllWorkersUpgraded), + upgradesteps.Action(string(upgradev1alpha1.RemoveMaintWindow), u.RemoveMaintWindow), + upgradesteps.Action(string(upgradev1alpha1.PostClusterHealthCheck), u.PostUpgradeHealthCheck), + upgradesteps.Action(string(upgradev1alpha1.SendCompletedNotification), u.SendCompletedNotification), + } +} + +// UpgradeCluster performs the upgrade of the cluster and returns an indication of the +// last-executed upgrade phase and any error associated with the phase execution. +func (u *bmUpgrader) UpgradeCluster(ctx context.Context, upgradeConfig *upgradev1alpha1.UpgradeConfig, logger logr.Logger) (upgradev1alpha1.UpgradePhase, error) { + u.upgradeConfig = upgradeConfig + return u.runSteps(ctx, logger, u.steps) +} + +// HealthCheck performs a pre-upgrade healthcheck when an upgrade is scheduled in advance mainly +// to highlight and notify of issues which could get fixed before the upgrade begins. +func (u *bmUpgrader) HealthCheck(ctx context.Context, upgradeConfig *upgradev1alpha1.UpgradeConfig, logger logr.Logger) (bool, error) { + u.upgradeConfig = upgradeConfig + ok, err := u.PreUpgradeHealthCheck(ctx, logger) + return ok, err +} diff --git a/pkg/upgraders/builder.go b/pkg/upgraders/builder.go index 861cbb68..3165c605 100644 --- a/pkg/upgraders/builder.go +++ b/pkg/upgraders/builder.go @@ -49,6 +49,12 @@ func (cub *clusterUpgraderBuilder) NewClient(c client.Client, cfm configmanager. return nil, err } return cu, nil + case upgradev1alpha1.BM: + cu, err := NewBMUpgrader(c, cfm, mc, nc) + if err != nil { + return nil, err + } + return cu, nil default: cu, err := NewOSDUpgrader(c, cfm, mc, nc) if err != nil {