diff --git a/README.md b/README.md index f337c25..434cf6a 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ flowchart LR subgraph FS[flux-system namespace] fluxctrl[Flux Controllers] pullsecret([image-pull-secret]) + caconfigmap([custom-ca-configmap]) end end @@ -45,6 +46,7 @@ flowchart LR spflux -- creates --> helmrel helmrel -- installs --> fluxctrl spflux -- copies secrets --> pullsecret + spflux -- copies configmap --> caconfigmap mcpapi -- represents --> mcp ``` @@ -146,6 +148,10 @@ metadata: spec: # Optional: Reconciliation interval pollInterval: "5m" + # Optional: ConfigMapKeySelector for a custom ca bundle (configmap will be copied to ManagedControlPlane) + caBundleRef: + name: "custom-ca-bundle" + key: "ca-bundle.crt" # The Flux versions that can be installed versions: - version: "2.8.3" @@ -170,12 +176,21 @@ spec: tag: v1.8.1 ``` -| Field | Type | Description | -| ------------------- | -------- | ---------------------------------------------- | -| `spec.pollInterval` | duration | How often to reconcile resources (default: 1m) | -| `spec.versions` | array | The versions of Flux that can be installed | +| Field | Type | Description | +| ------------------- | -------------------- | ------------------------------------------------------------------ | +| `spec.pollInterval` | duration | How often to reconcile resources (default: 1m) | +| `spec.certSecretRef`| object | SecretRef for chart registry trust establishment | +| `spec.caBundleRef` | ConfigMapKeySelector | A configmap with a ca bundle used by Flux to verify certificates | +| `spec.versions` | array | The versions of Flux that can be installed | -A version item is defined as follows: +A **caBundleRef** is defined as follows: + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | The name of the configmap which holds the ca bundle | +| `key` | string | The key in the configmap under which the ca bundle is stored | + +A **version** item is defined as follows: | Field | Type | Description | | ----------------- | ------ | --------------------------------------------- | @@ -210,7 +225,7 @@ For air-gapped or enterprise environments, see the [Image Localization Guide](do | Status reporting & error messages | ✅ | | | Operation annotations | ⚠️ | `openmcp.cloud/operation: ignore` is processed by [opencontrolplane-runtime](https://github.com/openmcp-project/opencontrolplane-runtime). `openmcp.cloud/operation: reconcile` is not processed. | | API stability policy | ✅ | | -| Custom CA support | ⚠️ | Private-registry pull secrets supported; custom CA bundle propagation to Flux components is not implemented. | +| Custom CA support | ✅ | | | Release artifacts (image + OCM) | ✅ | | | Testing | ✅ | | | Ownership and maintenance docs | ✅ | | diff --git a/api/crds/manifests/flux.services.open-control-plane.io_providerconfigs.yaml b/api/crds/manifests/flux.services.open-control-plane.io_providerconfigs.yaml index a228b44..ce91c6e 100644 --- a/api/crds/manifests/flux.services.open-control-plane.io_providerconfigs.yaml +++ b/api/crds/manifests/flux.services.open-control-plane.io_providerconfigs.yaml @@ -41,6 +41,31 @@ spec: spec: description: spec defines the desired state of ProviderConfig properties: + caBundleRef: + description: |- + CABundleRef is a reference to a config map containing a certificate bundle. + It will be installed on the ManagedControlPlane and configured for the Flux controllers. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic pollInterval: default: 1m description: PollInterval determines how often to reconcile resources diff --git a/api/v1alpha1/providerconfig_types.go b/api/v1alpha1/providerconfig_types.go index f9c1d4e..0c33c37 100644 --- a/api/v1alpha1/providerconfig_types.go +++ b/api/v1alpha1/providerconfig_types.go @@ -19,6 +19,7 @@ package v1alpha1 import ( "time" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -35,6 +36,11 @@ type ProviderConfigSpec struct { // +kubebuilder:default:="1m" // +kubebuilder:validation:Format=duration PollInterval *metav1.Duration `json:"pollInterval,omitempty"` + + // CABundleRef is a reference to a config map containing a certificate bundle. + // It will be installed on the ManagedControlPlane and configured for the Flux controllers. + // +kubebuilder:validation:Optional + CABundleRef *corev1.ConfigMapKeySelector `json:"caBundleRef,omitempty"` } // FluxVersion defines a version of Flux that can be installed diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index cadbb0b..c55646d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -5,6 +5,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -228,6 +229,11 @@ func (in *ProviderConfigSpec) DeepCopyInto(out *ProviderConfigSpec) { *out = new(v1.Duration) **out = **in } + if in.CABundleRef != nil { + in, out := &in.CABundleRef, &out.CABundleRef + *out = new(corev1.ConfigMapKeySelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderConfigSpec. diff --git a/docs/configuration/image-localization.md b/docs/configuration/image-localization.md index ee1759b..1606ef2 100644 --- a/docs/configuration/image-localization.md +++ b/docs/configuration/image-localization.md @@ -9,12 +9,14 @@ In air-gapped environments, you typically need to: 1. **Mirror the Flux Helm chart** to your internal OCI registry 2. **Mirror Flux controller images** to your internal container registry 3. **Configure authentication** for both chart and image pulls +4. **Add custom CA certificates** if you use private PKI with self-signed certificates The Flux service provider handles this through: - **`chartPullSecret`**: Credentials for pulling the Helm chart from a private OCI registry - **`values.imagePullSecrets`**: Credentials for pulling Flux controller images (specified in Helm values) - **`values`**: Custom Helm values for image location overrides +- **`caBundleRef`**: PEM-encoded custom ca bundle if your private OCI registry uses self-signed certificates ## Secret Flow @@ -24,6 +26,7 @@ flowchart TB subgraph PC[Platform Cluster] subgraph SPN[Service Provider Namespace] chartsecret([chart-pull-secret]) + caconfigmap([custom-ca-configmap]) imgsecret1([image-pull-secret-1]) imgsecret2([image-pull-secret-2]) end @@ -40,15 +43,18 @@ flowchart TB subgraph FS[flux-system namespace] imgcopy1([image-pull-secret-1]) imgcopy2([image-pull-secret-2]) + caconfigmapcopy([custom-ca-configmap]) fluxctrl[Flux Controllers] fluxctrl -. uses .-> imgcopy1 fluxctrl -. uses .-> imgcopy2 + fluxctrl -. uses .-> caconfigmapcopy end end chartsecret -- copied to --> chartsecretcopy imgsecret1 -- copied to --> imgcopy1 imgsecret2 -- copied to --> imgcopy2 + caconfigmap -- copied to --> caconfigmapcopy helmrel -- installs --> fluxctrl ``` @@ -62,6 +68,13 @@ kind: ProviderConfig metadata: name: flux-provider-config spec: + # ConfigMapKeySelector pointing to a configmap which holds a PEM-encoded custom CA bundle. + # Must exist in the service provider's namespace on the platform cluster + # The configmap will be automatically copied from the service provider's namespace + # and configured for the Flux controllers. + caBundleRef: + name: "custom-ca-bundle" + key: "ca-bundle.crt" versions: - version: "2.8.3" chartVersion: "2.18.2" @@ -112,6 +125,33 @@ kubectl create secret docker-registry image-registry-credentials \ --docker-password= ``` +### Creating Custom CA ConfigMap + +Concatenate all your custom CA certificates into a single PEM file. Each certificate must use the standard PEM format. + +```shell +cat /path/to/ca1.crt /path/to/ca2.crt > ca-bundle.crt +``` + +The resulting file should look like this: + +```text title="ca-bundle.crt" +-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAMSO... +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJANPQ... +-----END CERTIFICATE----- +``` + +Create the configmap + +```shell +kubectl create configmap custom-ca-bundle \ + --from-file=ca-bundle.crt=ca-bundle.crt \ + --namespace=openmcp-system +``` + ## How It Works ### Chart Pull Secret @@ -126,6 +166,14 @@ kubectl create secret docker-registry image-registry-credentials \ 2. These secrets are copied from the service provider's namespace on the platform cluster to `flux-system` on the ManagedControlPlane 3. The Helm values are passed through to Flux, which configures the controller pods with these secrets +### Custom CA Bundle +1. The configmap specified in `caBundleRef` is copied from the service provider's namespace on the platform cluster to `flux-system` on the ManagedControlPlane +2. For each Flux controller, the Helm values are adjusted so that it mounts the provided `caBundleRef.key` and sets the `SSL_CERT_DIR` environment variable to add the bundle to the pool of known certificates +3. The Helm values are passed through to Flux, and each Flux controller is able to verify certificates signed by the provided custom CA + +[!CAUTION] +The custom CA certificate is not propagated to the OpenControlPlane cluster nodes. If you want to pull images from the same OCI registry you must add the custom CA certificate to the cluster nodes yourself. + ## Complete Example ### Air-Gapped Setup @@ -138,6 +186,9 @@ metadata: spec: chartUrl: "oci://harbor.corp.internal/charts/flux2" chartPullSecret: "harbor-credentials" + caBundleRef: + name: "harbor-ca-bundle" + key: "harbor-ca-bundle.crt" values: # Image pull secrets - will be copied to ManagedControlPlane imagePullSecrets: @@ -192,6 +243,14 @@ kubectl get secrets -n mcp-- | grep -E "chart|image" kubectl get secrets -n flux-system | grep -E "image" ``` +### Check ConfigMap Copying +Verify configmaps are copied to the correct namespaces: + +```bash +# ManagedControlPlane - flux-system namespace +kubectl get cm -n flux-system | grep -E "ca" +``` + ### Check OCIRepository Secret Reference ```bash diff --git a/internal/controller/flux_controller.go b/internal/controller/flux_controller.go index 7e1b1d8..92b1a60 100644 --- a/internal/controller/flux_controller.go +++ b/internal/controller/flux_controller.go @@ -42,7 +42,7 @@ import ( const conditionReasonError = "ReconcileError" // ErrManagedResources is an end-user facing error if errors are present inside Flux.Status.ManagedResources -var ErrManagedResources error = errors.New("resources contain reconcile errors") +var ErrManagedResources = errors.New("resources contain reconcile errors") // FluxReconciler reconciles a Flux object type FluxReconciler struct { @@ -117,6 +117,9 @@ func userErrorMessage(err error) string { if errors.Is(err, flux.ErrSecretCleanup) { errorMessages = append(errorMessages, flux.ErrSecretCleanup.Error()) } + if errors.Is(err, flux.ErrConfigMapCleanup) { + errorMessages = append(errorMessages, flux.ErrConfigMapCleanup.Error()) + } return strings.Join(errorMessages, "; ") } @@ -172,6 +175,23 @@ func (r *FluxReconciler) createObjectManager(obj *apiv1alpha1.Flux, pc *apiv1alp }) } + if pc.Spec.CABundleRef != nil { + // add custom ca volume, volumeMount and envVar to helm values + fluxVersion.Values, err = flux.AddCAToHelmValues(fluxVersion.Values, pc.Spec.CABundleRef) + if err != nil { + return nil, fmt.Errorf("failed to add ca volume to helm values: %w", err) + } + + // Sync ca configmap from platform cluster to MCP + flux.ManageCaConfigMap(mcpCluster, pc.Spec.CABundleRef.LocalObjectReference, flux.ConfigMapCopyConfig{ + SourceClient: r.PlatformCluster.Client(), + SourceNamespace: r.PodNamespace, + TargetNamespace: fluxNamespace, + TargetName: flux.CustomCABundleConfigMapName, + }) + + } + // Configure Flux resources (OCIRepository and HelmRelease) flux.ManageFluxResources(flux.ManageFluxResourcesParams{ Cluster: platformCluster, @@ -194,10 +214,18 @@ func (r *FluxReconciler) createObjectManager(obj *apiv1alpha1.Flux, pc *apiv1alp Name: prefixedChartPullSecret, }, }) - controlPlaneCleaner := flux.NewSecretCleaner(mcpCluster, fluxNamespace, helmValues.ImagePullSecrets) + controlPlaneSecretCleaner := flux.NewSecretCleaner(mcpCluster, fluxNamespace, helmValues.ImagePullSecrets) + + configMapsToKeep := []corev1.LocalObjectReference{} + if pc.Spec.CABundleRef != nil { + configMapsToKeep = append(configMapsToKeep, corev1.LocalObjectReference{Name: flux.CustomCABundleConfigMapName}) + } + + controlPlaneConfigMapCleaner := flux.NewConfigMapCleaner(mcpCluster, fluxNamespace, configMapsToKeep) mgr.AddCleaner(platformCleaner) - mgr.AddCleaner(controlPlaneCleaner) + mgr.AddCleaner(controlPlaneSecretCleaner) + mgr.AddCleaner(controlPlaneConfigMapCleaner) return mgr, nil } diff --git a/internal/controller/flux_controller_test.go b/internal/controller/flux_controller_test.go index 695b021..8626725 100644 --- a/internal/controller/flux_controller_test.go +++ b/internal/controller/flux_controller_test.go @@ -296,19 +296,33 @@ func Test_updateStatusError(t *testing.T) { wantMessage: ErrManagedResources.Error(), }, { - name: "cleanup error", + name: "secret cleanup error", obj: &apiv1alpha1.Flux{}, resourceErrors: false, err: flux.ErrSecretCleanup, wantMessage: flux.ErrSecretCleanup.Error(), }, { - name: "combined resource and cleanup error", + name: "combined resource and secret cleanup error", obj: &apiv1alpha1.Flux{}, resourceErrors: true, err: flux.ErrSecretCleanup, wantMessage: fmt.Sprintf("%s; %s", ErrManagedResources.Error(), flux.ErrSecretCleanup.Error()), }, + { + name: "configmap cleanup error", + obj: &apiv1alpha1.Flux{}, + resourceErrors: false, + err: flux.ErrConfigMapCleanup, + wantMessage: flux.ErrConfigMapCleanup.Error(), + }, + { + name: "combined resource and configmap cleanup error", + obj: &apiv1alpha1.Flux{}, + resourceErrors: true, + err: flux.ErrConfigMapCleanup, + wantMessage: fmt.Sprintf("%s; %s", ErrManagedResources.Error(), flux.ErrConfigMapCleanup.Error()), + }, { name: "resource error and no end-user error", obj: &apiv1alpha1.Flux{}, diff --git a/pkg/flux/configmap.go b/pkg/flux/configmap.go new file mode 100644 index 0000000..d93a542 --- /dev/null +++ b/pkg/flux/configmap.go @@ -0,0 +1,163 @@ +// Copyright 2025. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flux + +import ( + "context" + "errors" + "fmt" + "slices" + + openmcpresources "github.com/openmcp-project/controller-utils/pkg/resources" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + apiv1alpha1 "github.com/openmcp-project/service-provider-flux/api/v1alpha1" +) + +// ErrConfigMapCleanup is an user-facing error that indicates configmap cleanup failures +var ErrConfigMapCleanup = errors.New("configmap cleanup failed") + +// ConfigMapCopyConfig holds the configuration for copying configmap. +type ConfigMapCopyConfig struct { + // SourceClient is the client to read the source configmap from. + SourceClient client.Client + // SourceNamespace is the namespace of the source configmap. + SourceNamespace string + // TargetNamespace is the namespace of the target configmap. + TargetNamespace string + // TargetName is an optional value to adjust the name of the target configmap + // instead of using the source configmap name. + TargetName string +} + +// ManageCaConfigMap syncs the ca configmap to the target cluster. +func ManageCaConfigMap(targetCluster ManagedCluster, caConfigMap corev1.LocalObjectReference, config ConfigMapCopyConfig) { + caConfigMapName := caConfigMap.Name + if config.TargetName != "" { + caConfigMapName = config.TargetName + } + configMap := NewManagedObject(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: caConfigMapName, + Namespace: config.TargetNamespace, + }, + }, ManagedObjectContext{ + ReconcileFunc: func(ctx context.Context, o client.Object) error { + oConfigMap, ok := o.(*corev1.ConfigMap) + if !ok { + return fmt.Errorf("expected *corev1.ConfigMap, got %T", o) + } + sourceConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: caConfigMap.Name, + Namespace: config.SourceNamespace, + }, + } + // retrieve source configmap from platform cluster + if err := config.SourceClient.Get(ctx, client.ObjectKeyFromObject(sourceConfigMap), sourceConfigMap); err != nil { + return err + } + mutator := openmcpresources.NewConfigMapMutator(caConfigMapName, config.TargetNamespace, sourceConfigMap.Data) + return mutator.Mutate(oConfigMap) + }, + StatusFunc: ConfigMapStatus, + }) + targetCluster.AddObject(configMap) +} + +// ConfigMapStatus returns the status of a configmap object. +func ConfigMapStatus(o client.Object, rl apiv1alpha1.ResourceLocation) Status { + if !o.GetDeletionTimestamp().IsZero() { + return Status{ + Phase: apiv1alpha1.Terminating, + Message: "ConfigMap is terminating.", + Location: rl, + } + } + if o.GetUID() == "" { + return Status{ + Phase: apiv1alpha1.Pending, + Message: "ConfigMap has not been created yet.", + Location: rl, + } + } + return Status{ + Phase: apiv1alpha1.Ready, + Message: "ConfigMap exists.", + Location: rl, + } +} + +var _ OrphanCleaner = &configMapCleaner{} + +type configMapCleaner struct { + cluster ManagedCluster + namespace string + configMapsToKeep []corev1.LocalObjectReference +} + +// NewConfigMapCleaner removes redundant configmaps in the given target namespace +// by removing any configmap labeled as managed by sp-flux that is not in configMapsToKeep. +func NewConfigMapCleaner(cluster ManagedCluster, namespace string, configMapsToKeep []corev1.LocalObjectReference) OrphanCleaner { + return &configMapCleaner{ + cluster: cluster, + namespace: namespace, + configMapsToKeep: configMapsToKeep, + } +} + +func (c *configMapCleaner) Cleanup(ctx context.Context) ([]Result, error) { + results := []Result{} + configMapCopies := &corev1.ConfigMapList{} + cl := c.cluster.GetClient() + if err := cl.List(ctx, configMapCopies, + client.InNamespace(c.namespace), + client.MatchingLabels{LabelManagedBy: labelServiceProviderFlux}, + ); err != nil { + log.FromContext(ctx).Error(err, "failed to list configmap for orphan cleanup") + return nil, ErrConfigMapCleanup + } + for _, configMap := range configMapCopies.Items { + if !slices.ContainsFunc(c.configMapsToKeep, func(ref corev1.LocalObjectReference) bool { return configMap.Name == ref.Name }) { + if err := cl.Delete(ctx, &configMap); client.IgnoreNotFound(err) != nil { + results = append(results, c.cleanupErrorResult(&configMap, err)) + } + } + } + return results, nil +} + +func (c *configMapCleaner) cleanupErrorResult(obj *corev1.ConfigMap, err error) Result { + return Result{ + Object: &managedObject{ + object: obj, + statusFunc: cleanupErrorStatusConfigMap, + deletionPolicy: Delete, + }, + Cluster: c.cluster, + OperationResult: OperationResultDeletionFailed, + Error: err, + } +} + +func cleanupErrorStatusConfigMap(_ client.Object, rl apiv1alpha1.ResourceLocation) Status { + return Status{ + Phase: apiv1alpha1.Terminating, + Message: "ConfigMap cleanup failed", + Location: rl, + } +} diff --git a/pkg/flux/configmap_test.go b/pkg/flux/configmap_test.go new file mode 100644 index 0000000..65b73b3 --- /dev/null +++ b/pkg/flux/configmap_test.go @@ -0,0 +1,288 @@ +// Copyright 2025. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flux + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1alpha1 "github.com/openmcp-project/service-provider-flux/api/v1alpha1" + + "github.com/openmcp-project/service-provider-flux/pkg/testutils" +) + +func TestManageCaConfigMap(t *testing.T) { + sourceConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-ca", + Namespace: "source-ns", + }, + Data: map[string]string{ + "ca.crt": "CERTDATA", + }, + } + fakeCluster := testutils.CreateFakeCluster(t, "platform", sourceConfigMap) + + tests := []struct { + name string + targetCluster ManagedCluster + caConfigMap corev1.LocalObjectReference + config ConfigMapCopyConfig + }{ + { + name: "syncs configmap", + targetCluster: NewManagedCluster(fakeCluster, &rest.Config{}, "target-ns", ManagedControlPlane), + caConfigMap: corev1.LocalObjectReference{Name: "custom-ca"}, + config: ConfigMapCopyConfig{ + SourceClient: fakeCluster.Client(), + SourceNamespace: "source-ns", + TargetNamespace: "target-ns", + }, + }, + { + name: "syncs configmap with target name adjustment", + targetCluster: NewManagedCluster(fakeCluster, &rest.Config{}, "target-ns", ManagedControlPlane), + caConfigMap: corev1.LocalObjectReference{Name: "custom-ca"}, + config: ConfigMapCopyConfig{ + SourceClient: fakeCluster.Client(), + SourceNamespace: "source-ns", + TargetNamespace: "target-ns", + TargetName: "sp-flux-custom-ca", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ManageCaConfigMap(tt.targetCluster, tt.caConfigMap, tt.config) + + mgr := NewManager() + mgr.AddCluster(tt.targetCluster) + results, gotErr := mgr.Apply(context.Background()) + require.NoError(t, gotErr) + for _, r := range results { + require.NoError(t, r.Error) + } + + targetConfigMap := &corev1.ConfigMap{} + targetName := tt.caConfigMap.Name + if tt.config.TargetName != "" { + targetName = tt.config.TargetName + } + err := fakeCluster.Client().Get(context.Background(), client.ObjectKey{ + Name: targetName, + Namespace: tt.config.TargetNamespace, + }, targetConfigMap) + require.NoError(t, err) + assert.Equal(t, sourceConfigMap.Data, targetConfigMap.Data) + }) + } +} + +func Test_configMapCleaner_Cleanup(t *testing.T) { + tests := []struct { + name string + cluster ManagedCluster + targetNamespace string + configMapsToKeep []corev1.LocalObjectReference + want []corev1.ConfigMap + wantResults bool + wantErr bool + }{ + { + name: "only managed configmaps are deleted", + targetNamespace: "flux-system", + cluster: createFakeCluster(createFakeClient([]client.Object{ + testConfigMap("a", "flux-system", true), + testConfigMap("b", "flux-system", false), + })), + configMapsToKeep: []corev1.LocalObjectReference{}, + want: []corev1.ConfigMap{ + *testConfigMap("b", "flux-system", false), + }, + wantErr: false, + }, + { + name: "configmaps in other namespaces are not deleted", + targetNamespace: "openmcp-system", + cluster: createFakeCluster(createFakeClient([]client.Object{ + testConfigMap("a", "flux-system", true), + testConfigMap("b", "flux-system", false), + })), + configMapsToKeep: []corev1.LocalObjectReference{}, + want: []corev1.ConfigMap{ + *testConfigMap("a", "flux-system", true), + *testConfigMap("b", "flux-system", false), + }, + wantErr: false, + }, + { + name: "configmaps to keep are not deleted", + targetNamespace: "flux-system", + cluster: createFakeCluster(createFakeClient([]client.Object{ + testConfigMap("a", "flux-system", true), + testConfigMap("b", "flux-system", false), + })), + configMapsToKeep: []corev1.LocalObjectReference{ + { + Name: "a", + }, + }, + want: []corev1.ConfigMap{ + *testConfigMap("a", "flux-system", true), + *testConfigMap("b", "flux-system", false), + }, + wantErr: false, + }, + { + name: "error is returned when list fails", + cluster: createFakeCluster(listErrorClient{}), + targetNamespace: "flux-system", + configMapsToKeep: []corev1.LocalObjectReference{}, + want: []corev1.ConfigMap{}, + wantErr: true, + }, + { + name: "error is returned when delete fails", + cluster: createFakeCluster(deleteErrorConfigMapClient{ + fakeConfigMap: *testConfigMap("a", "flux-system", true), + }), + targetNamespace: "flux-system", + configMapsToKeep: []corev1.LocalObjectReference{}, + want: []corev1.ConfigMap{}, + wantErr: false, + wantResults: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewConfigMapCleaner(tt.cluster, tt.targetNamespace, tt.configMapsToKeep) + results, gotErr := c.Cleanup(context.Background()) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("Cleanup() failed: %v", gotErr) + } + return + } + if len(results) > 0 { + if !tt.wantResults { + t.Errorf("Cleanup() failed %v", results) + } + return + } + configMapList := &corev1.ConfigMapList{} + require.NoError(t, tt.cluster.GetClient().List(context.Background(), configMapList)) + for _, gotConfigMap := range configMapList.Items { + assert.True(t, slices.ContainsFunc(tt.want, func(cm corev1.ConfigMap) bool { + return cm.Name == gotConfigMap.Name && cm.Namespace == gotConfigMap.Namespace + })) + } + }) + } +} + +func testConfigMap(name, namespace string, managedByFlux bool) *corev1.ConfigMap { + labels := map[string]string{} + if managedByFlux { + labels[LabelManagedBy] = labelServiceProviderFlux + } + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + } +} + +// TestConfigMapStatus tests the ConfigMapStatus function +func TestConfigMapStatus(t *testing.T) { + tests := []struct { + name string + obj client.Object + rl apiv1alpha1.ResourceLocation + expected apiv1alpha1.InstancePhase + }{ + { + name: "configmap with UID - ready", + obj: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + UID: "test-uid", + }, + }, + rl: apiv1alpha1.ManagedControlPlane, + expected: apiv1alpha1.Ready, + }, + { + name: "configmap without UID - pending", + obj: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + }, + }, + rl: apiv1alpha1.ManagedControlPlane, + expected: apiv1alpha1.Pending, + }, + { + name: "configmap being deleted - terminating", + obj: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + UID: "test-uid", + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{"test-finalizer"}, + }, + }, + rl: apiv1alpha1.ManagedControlPlane, + expected: apiv1alpha1.Terminating, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := ConfigMapStatus(tt.obj, tt.rl) + assert.Equal(t, tt.expected, status.Phase) + assert.Equal(t, tt.rl, status.Location) + }) + } +} + +type deleteErrorConfigMapClient struct { + client.Client + fakeConfigMap corev1.ConfigMap +} + +func (d deleteErrorConfigMapClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + configMapList := list.(*corev1.ConfigMapList) + configMapList.Items = []corev1.ConfigMap{d.fakeConfigMap} + return nil +} + +func (d deleteErrorConfigMapClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return errors.New("delete failed") +} diff --git a/pkg/flux/flux_test.go b/pkg/flux/flux_test.go index ee43f27..cb2c440 100644 --- a/pkg/flux/flux_test.go +++ b/pkg/flux/flux_test.go @@ -172,62 +172,6 @@ func TestSimpleStatus(t *testing.T) { } } -// TestSecretStatus tests the SecretStatus function -func TestSecretStatus(t *testing.T) { - tests := []struct { - name string - obj client.Object - rl apiv1alpha1.ResourceLocation - expected apiv1alpha1.InstancePhase - }{ - { - name: "secret with UID - ready", - obj: &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "test-ns", - UID: "test-uid", - }, - }, - rl: apiv1alpha1.ManagedControlPlane, - expected: apiv1alpha1.Ready, - }, - { - name: "secret without UID - pending", - obj: &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "test-ns", - }, - }, - rl: apiv1alpha1.ManagedControlPlane, - expected: apiv1alpha1.Pending, - }, - { - name: "secret being deleted - terminating", - obj: &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "test-ns", - UID: "test-uid", - DeletionTimestamp: &metav1.Time{Time: time.Now()}, - Finalizers: []string{"test-finalizer"}, - }, - }, - rl: apiv1alpha1.ManagedControlPlane, - expected: apiv1alpha1.Terminating, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - status := SecretStatus(tt.obj, tt.rl) - assert.Equal(t, tt.expected, status.Phase) - assert.Equal(t, tt.rl, status.Location) - }) - } -} - // TestSetManagedBy tests the SetManagedBy function func TestSetManagedBy(t *testing.T) { tests := []struct { diff --git a/pkg/flux/helm.go b/pkg/flux/helm.go index e9a9e86..dfe230e 100644 --- a/pkg/flux/helm.go +++ b/pkg/flux/helm.go @@ -15,11 +15,42 @@ package flux import ( "encoding/json" + "errors" + "fmt" + "strings" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) +const ( + // customCaVolumeName is the name of the custom ca volume and volume mount + customCaVolumeName = "custom-ca-bundle" + + // customCaPath is the path the ca bundle will be mounted into + customCaPath = "/etc/open-control-plane/custom-ca" + + // CustomCABundleConfigMapName is the fixed name for the copied CA bundle ConfigMap on the MCP cluster. + CustomCABundleConfigMapName = "custom-ca-bundle" +) + +// certDirectories contains a list of places where the default system certs are stored in addition to caBundleMountDir +// from x509 go lib (https://github.com/golang/go/blob/015343854b5d9e2829481df30dbcae2ca6682d25/src/crypto/x509/root_linux.go) +var certDirectories = []string{ + "/etc/ssl/certs", + "/etc/pki/tls/certs", +} + +var fluxControllers = []string{ + "helmController", + "imageAutomationController", + "imageReflectionController", + "kustomizeController", + "notificationController", + "sourceController", + "sourceWatcher", +} + // HelmValues defines the Helm values that are explicitly processed during reconciliation. // The Flux Helm chart uses imagePullSecrets at the top level, not under global. type HelmValues struct { @@ -44,3 +75,180 @@ func ExtractHelmValues(values *apiextensionsv1.JSON) (*HelmValues, error) { return vals, nil } + +// AddCAToHelmValues removes conflicting volumes, volumeMounts and envVars (matching by name and/or mountPath) and +// adds a volume, volumeMount and envVar on all Flux controller helm values sections to import the custom CA certificate. +// nolint:gocyclo +func AddCAToHelmValues(values *apiextensionsv1.JSON, configMap *corev1.ConfigMapKeySelector) (*apiextensionsv1.JSON, error) { + if configMap == nil { + return nil, errors.New("cannot add custom CA to Helm values: ConfigMapKeySelector is nil") + } + + if configMap.Name == "" { + return nil, errors.New("cannot add custom CA to Helm values: caBundleRef.Name must be set") + } + + if configMap.Key == "" { + return nil, errors.New("cannot add custom CA to Helm values: caBundleRef.Key must be set") + } + + var root = map[string]json.RawMessage{} + + caVolume := corev1.Volume{ + Name: customCaVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: CustomCABundleConfigMapName, + }, + Items: []corev1.KeyToPath{ + { + Key: configMap.Key, + Path: configMap.Key, + }, + }, + }, + }, + } + + caVolumeMount := corev1.VolumeMount{ + Name: customCaVolumeName, + ReadOnly: true, + MountPath: customCaPath, + } + + caEnvVar := corev1.EnvVar{ + Name: "SSL_CERT_DIR", + Value: strings.Join(append(certDirectories, customCaPath), ":"), + } + + if values != nil && len(values.Raw) > 0 { + if err := json.Unmarshal(values.Raw, &root); err != nil { + return nil, fmt.Errorf("failed to unmarshal helm values: %w", err) + } + if root == nil { + root = make(map[string]json.RawMessage) + } + } + + for _, controller := range fluxControllers { + if err := addCAToController(root, controller, caVolume, caVolumeMount, caEnvVar); err != nil { + return nil, err + } + } + + out, err := json.Marshal(root) + if err != nil { + return nil, fmt.Errorf("failed to marshal helm values: %w", err) + } + + return &apiextensionsv1.JSON{Raw: out}, nil +} + +func addCAToController( + root map[string]json.RawMessage, + controller string, + caVolume corev1.Volume, + caVolumeMount corev1.VolumeMount, + caEnvVar corev1.EnvVar, +) error { + var controllerValues map[string]json.RawMessage + var volumes []corev1.Volume + var volumeMounts []corev1.VolumeMount + var envVars []corev1.EnvVar + + if err := unmarshalIfPresent(root, controller, &controllerValues); err != nil { + return fmt.Errorf("failed to unmarshal %s: %w", controller, err) + } + + if controllerValues == nil { + controllerValues = map[string]json.RawMessage{} + } + + if err := unmarshalIfPresent(controllerValues, "volumes", &volumes); err != nil { + return fmt.Errorf("failed to unmarshal %s.volumes: %w", controller, err) + } + + if err := unmarshalIfPresent(controllerValues, "volumeMounts", &volumeMounts); err != nil { + return fmt.Errorf("failed to unmarshal %s.volumeMounts: %w", controller, err) + } + + if err := unmarshalIfPresent(controllerValues, "extraEnv", &envVars); err != nil { + return fmt.Errorf("failed to unmarshal %s.extraEnv: %w", controller, err) + } + + volumes = removeConflictingVolumesAndAppend(volumes, caVolume) + volumeMounts = removeConflictingVolumeMountsAndAppend(volumeMounts, caVolumeMount) + envVars = removeConflictingEnvVarsAndAppend(envVars, caEnvVar) + + volumesRaw, err := json.Marshal(volumes) + if err != nil { + return fmt.Errorf("failed to marshal %s.volumes: %w", controller, err) + } + + volumeMountsRaw, err := json.Marshal(volumeMounts) + if err != nil { + return fmt.Errorf("failed to marshal %s.volumeMounts: %w", controller, err) + } + + envVarsRaw, err := json.Marshal(envVars) + if err != nil { + return fmt.Errorf("failed to marshal %s.extraEnv: %w", controller, err) + } + + controllerValues["volumes"] = volumesRaw + controllerValues["volumeMounts"] = volumeMountsRaw + controllerValues["extraEnv"] = envVarsRaw + + controllerRaw, err := json.Marshal(controllerValues) + if err != nil { + return fmt.Errorf("failed to marshal %s: %w", controller, err) + } + + root[controller] = controllerRaw + return nil +} + +func removeConflictingVolumesAndAppend(volumes []corev1.Volume, caVolume corev1.Volume) []corev1.Volume { + updated := []corev1.Volume{} + for _, volume := range volumes { + if volume.Name != caVolume.Name { + updated = append(updated, volume) + } + } + updated = append(updated, caVolume) + return updated +} + +func removeConflictingVolumeMountsAndAppend(volumeMounts []corev1.VolumeMount, caVolumeMount corev1.VolumeMount) []corev1.VolumeMount { + updated := []corev1.VolumeMount{} + for _, volumeMount := range volumeMounts { + if volumeMount.MountPath != caVolumeMount.MountPath && volumeMount.Name != caVolumeMount.Name { + updated = append(updated, volumeMount) + } + } + updated = append(updated, caVolumeMount) + return updated +} + +func removeConflictingEnvVarsAndAppend(envVars []corev1.EnvVar, caEnvVar corev1.EnvVar) []corev1.EnvVar { + updated := []corev1.EnvVar{} + for _, envVar := range envVars { + if envVar.Name != caEnvVar.Name { + updated = append(updated, envVar) + } + } + updated = append(updated, caEnvVar) + return updated +} + +func unmarshalIfPresent(obj map[string]json.RawMessage, key string, out any) error { + raw, ok := obj[key] + if !ok || len(raw) == 0 { + return nil + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("invalid %s JSON: %w", key, err) + } + return nil +} diff --git a/pkg/flux/helm_test.go b/pkg/flux/helm_test.go index a91dca7..24e4632 100644 --- a/pkg/flux/helm_test.go +++ b/pkg/flux/helm_test.go @@ -15,10 +15,12 @@ package flux import ( "encoding/json" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) @@ -130,6 +132,335 @@ func TestExtractHelmValues(t *testing.T) { } } +func TestAddCaToHelmValues(t *testing.T) { + caBundleRef := &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-ca-configmap"}, + Key: "ca.crt", + } + + expectedCaVolume := corev1.Volume{ + Name: customCaVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: CustomCABundleConfigMapName}, + Items: []corev1.KeyToPath{{ + Key: "ca.crt", + Path: "ca.crt", + }}, + }, + }, + } + expectedCaVolumeMount := corev1.VolumeMount{ + Name: customCaVolumeName, + ReadOnly: true, + MountPath: customCaPath, + } + expectedCaEnvVar := corev1.EnvVar{ + Name: "SSL_CERT_DIR", + Value: strings.Join(append(certDirectories, customCaPath), ":"), + } + + tests := []struct { + name string + caBundleRef *corev1.ConfigMapKeySelector + values *apiextensionsv1.JSON + wantErr string + checkValue func(t *testing.T, out *apiextensionsv1.JSON) + }{ + { + name: "Adds controller volumes, volumeMounts and extraEnv when no helm values are set", + values: nil, + checkValue: func(t *testing.T, out *apiextensionsv1.JSON) { + require.NotNil(t, out) + + expected := buildHelmValues(t, + withAllControllerVolumes(expectedCaVolume), + withAllControllerVolumeMounts(expectedCaVolumeMount), + withAllControllerExtraEnv(expectedCaEnvVar), + ) + + assert.JSONEq(t, string(expected.Raw), string(out.Raw)) + }, + }, + { + name: "Preserves existing helm values and adds CA entries", + values: buildHelmValues(t, + withRootField("namespace", "other-namespace"), + withAllControllerField( + "resources", map[string]any{ + "limits": map[string]any{ + "memory": "256Mi", + }, + }), + withAllControllerVolumes(corev1.Volume{Name: "existing-volume"}), + withAllControllerVolumeMounts( + corev1.VolumeMount{Name: "existing-volume", MountPath: "/tmp/existing"}), + ), + checkValue: func(t *testing.T, out *apiextensionsv1.JSON) { + require.NotNil(t, out) + + expected := buildHelmValues(t, + withRootField("namespace", "other-namespace"), + withAllControllerField("resources", map[string]any{ + "limits": map[string]any{"memory": "256Mi"}, + }), + withAllControllerVolumes(expectedCaVolume), + withAllControllerVolumeMounts(expectedCaVolumeMount), + withAllControllerExtraEnv(expectedCaEnvVar), + withAllControllerVolumes( + corev1.Volume{Name: "existing-volume"}, + expectedCaVolume, + ), + withAllControllerVolumeMounts( + corev1.VolumeMount{Name: "existing-volume", MountPath: "/tmp/existing"}, + expectedCaVolumeMount, + ), + ) + assert.JSONEq(t, string(expected.Raw), string(out.Raw)) + }, + }, + { + name: "Removes VolumeMounts with same name and/or same MountPath", + values: buildHelmValues(t, + withAllControllerVolumeMounts( + corev1.VolumeMount{Name: "volume1", MountPath: "/tmp/volume1"}, + corev1.VolumeMount{Name: "volume2", MountPath: customCaPath}, + corev1.VolumeMount{Name: customCaVolumeName, MountPath: "/tmp/existing"}, + corev1.VolumeMount{Name: customCaVolumeName, MountPath: customCaPath}, + ), + ), + checkValue: func(t *testing.T, out *apiextensionsv1.JSON) { + require.NotNil(t, out) + + expected := buildHelmValues(t, + withAllControllerVolumes(expectedCaVolume), + withAllControllerExtraEnv(expectedCaEnvVar), + withAllControllerVolumeMounts( + corev1.VolumeMount{Name: "volume1", MountPath: "/tmp/volume1"}, + expectedCaVolumeMount, + ), + ) + + assert.JSONEq(t, string(expected.Raw), string(out.Raw)) + }, + }, + { + name: "Removes Volumes with same name", + values: buildHelmValues(t, + withAllControllerVolumes( + corev1.Volume{Name: customCaVolumeName}, + corev1.Volume{Name: "volume1"}, + ), + ), + checkValue: func(t *testing.T, out *apiextensionsv1.JSON) { + require.NotNil(t, out) + + expected := buildHelmValues(t, + withAllControllerVolumes( + corev1.Volume{Name: "volume1"}, + expectedCaVolume, + ), + withAllControllerVolumeMounts(expectedCaVolumeMount), + withAllControllerExtraEnv(expectedCaEnvVar), + ) + assert.JSONEq(t, string(expected.Raw), string(out.Raw)) + }, + }, + { + name: "Removes EnvVars with same name", + values: buildHelmValues(t, + withAllControllerExtraEnv( + corev1.EnvVar{Name: "SSL_CERT_DIR"}, + corev1.EnvVar{Name: "ANOTHER_ENV_VAR"}, + ), + ), + checkValue: func(t *testing.T, out *apiextensionsv1.JSON) { + require.NotNil(t, out) + + expected := buildHelmValues(t, + withAllControllerExtraEnv( + corev1.EnvVar{Name: "ANOTHER_ENV_VAR"}, + expectedCaEnvVar, + ), + withAllControllerVolumes(expectedCaVolume), + withAllControllerVolumeMounts(expectedCaVolumeMount), + ) + assert.JSONEq(t, string(expected.Raw), string(out.Raw)) + }, + }, + { + name: "null JSON value treated as empty", + values: &apiextensionsv1.JSON{Raw: []byte("null")}, + checkValue: func(t *testing.T, out *apiextensionsv1.JSON) { + require.NotNil(t, out) + + expected := buildHelmValues(t, + withAllControllerVolumes(expectedCaVolume), + withAllControllerVolumeMounts(expectedCaVolumeMount), + withAllControllerExtraEnv(expectedCaEnvVar), + ) + + assert.JSONEq(t, string(expected.Raw), string(out.Raw)) + }, + }, + { + name: "Returns error for invalid root json", + values: &apiextensionsv1.JSON{Raw: []byte("not-json")}, + wantErr: "failed to unmarshal helm values", + }, + { + name: "returns error for invalid controller json", + values: mustMarshalJSON(t, map[string]any{ + "helmController": "not-an-object", + }), + wantErr: "failed to unmarshal helmController", + }, + { + name: "returns error for invalid controller.volumes json", + values: mustMarshalJSON(t, map[string]any{ + "helmController": map[string]any{ + "volumes": "not-a-list", + }, + }), + wantErr: "failed to unmarshal helmController.volumes", + }, + { + name: "returns error for invalid controller.volumeMounts json", + values: mustMarshalJSON(t, map[string]any{ + "helmController": map[string]any{ + "volumeMounts": "not-a-list", + }, + }), + wantErr: "failed to unmarshal helmController.volumeMounts", + }, + { + name: "returns error for invalid controller.extraEnv json", + values: mustMarshalJSON(t, map[string]any{ + "helmController": map[string]any{ + "extraEnv": "not-a-list", + }, + }), + wantErr: "failed to unmarshal helmController.extraEnv", + }, + { + name: "returns error if configMap name is unset", + wantErr: "caBundleRef.Name must be set", + caBundleRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: ""}, + Key: "ca.crt", + }, + }, + { + name: "returns error if configMap key is unset", + wantErr: "caBundleRef.Key must be set", + caBundleRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-ca-configmap"}, + Key: "", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var out *apiextensionsv1.JSON + var err error + + if tt.caBundleRef != nil { + out, err = AddCAToHelmValues(tt.values, tt.caBundleRef) + } else { + out, err = AddCAToHelmValues(tt.values, caBundleRef) + } + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + + require.NoError(t, err) + require.NotNil(t, out) + if tt.checkValue != nil { + tt.checkValue(t, out) + } + }) + } +} + +type helmValues struct { + root map[string]any + controllers map[string]map[string]any +} + +type helmValuesOption func(*helmValues) + +func buildHelmValues(t *testing.T, opts ...helmValuesOption) *apiextensionsv1.JSON { + t.Helper() + + builder := &helmValues{ + root: map[string]any{}, + controllers: map[string]map[string]any{}, + } + + for _, opt := range opts { + opt(builder) + } + + for controller, values := range builder.controllers { + if len(values) > 0 { + builder.root[controller] = values + } + } + + return mustMarshalJSON(t, builder.root) +} + +func withRootField(key string, value any) helmValuesOption { + return func(builder *helmValues) { + builder.root[key] = value + } +} + +func withAllControllerField(key string, value any) helmValuesOption { + return func(builder *helmValues) { + for _, controller := range fluxControllers { + withControllerField(controller, key, value)(builder) + } + } +} + +func withControllerField(controller string, key string, value any) helmValuesOption { + return func(builder *helmValues) { + if _, ok := builder.controllers[controller]; !ok { + builder.controllers[controller] = map[string]any{} + } + builder.controllers[controller][key] = value + } +} + +func withAllControllerVolumes(volumes ...corev1.Volume) helmValuesOption { + return func(builder *helmValues) { + for _, controller := range fluxControllers { + withControllerField(controller, "volumes", volumes)(builder) + } + } +} + +func withAllControllerVolumeMounts(volumeMounts ...corev1.VolumeMount) helmValuesOption { + return func(builder *helmValues) { + for _, controller := range fluxControllers { + withControllerField(controller, "volumeMounts", volumeMounts)(builder) + } + } +} + +func withAllControllerExtraEnv(envVars ...corev1.EnvVar) helmValuesOption { + return func(builder *helmValues) { + for _, controller := range fluxControllers { + withControllerField(controller, "extraEnv", envVars)(builder) + } + } +} + func mustMarshalJSON(t *testing.T, v any) *apiextensionsv1.JSON { t.Helper() raw, err := json.Marshal(v) diff --git a/pkg/flux/secret.go b/pkg/flux/secret.go index 6bc6398..23fa6d5 100644 --- a/pkg/flux/secret.go +++ b/pkg/flux/secret.go @@ -78,7 +78,7 @@ func ManagePullSecrets(targetCluster ManagedCluster, imagePullSecrets []corev1.L mutator := openmcpresources.NewSecretMutator(secretName, config.TargetNamespace, sourceSecret.Data, corev1.SecretTypeDockerConfigJson) return mutator.Mutate(oSecret) }, - StatusFunc: SimpleStatus, + StatusFunc: SecretStatus, }) targetCluster.AddObject(secret) } diff --git a/pkg/flux/secret_test.go b/pkg/flux/secret_test.go index 4cba8a0..17e01e6 100644 --- a/pkg/flux/secret_test.go +++ b/pkg/flux/secret_test.go @@ -20,16 +20,16 @@ import ( "slices" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" + + apiv1alpha1 "github.com/openmcp-project/service-provider-flux/api/v1alpha1" "github.com/openmcp-project/service-provider-flux/pkg/testutils" ) @@ -233,24 +233,6 @@ func Test_secretCleaner_Cleanup(t *testing.T) { } } -var _ ManagedCluster = &fakeCluster{} - -type fakeCluster struct { - managedCluster - fakeClient client.Client -} - -// GetClient implements [ManagedCluster]. -func (f *fakeCluster) GetClient() client.Client { - return f.fakeClient -} - -func createFakeCluster(client client.Client) ManagedCluster { - return &fakeCluster{ - fakeClient: client, - } -} - func testSecret(name, namespace string, managedByFlux bool) *corev1.Secret { labels := map[string]string{} if managedByFlux { @@ -265,18 +247,60 @@ func testSecret(name, namespace string, managedByFlux bool) *corev1.Secret { } } -func createFakeClient(clusterObjects []client.Object) client.Client { - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - return fake.NewClientBuilder().WithObjects(clusterObjects...).WithScheme(scheme).Build() -} - -type listErrorClient struct { - client.Client -} +// TestSecretStatus tests the SecretStatus function +func TestSecretStatus(t *testing.T) { + tests := []struct { + name string + obj client.Object + rl apiv1alpha1.ResourceLocation + expected apiv1alpha1.InstancePhase + }{ + { + name: "secret with UID - ready", + obj: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + UID: "test-uid", + }, + }, + rl: apiv1alpha1.ManagedControlPlane, + expected: apiv1alpha1.Ready, + }, + { + name: "secret without UID - pending", + obj: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + }, + }, + rl: apiv1alpha1.ManagedControlPlane, + expected: apiv1alpha1.Pending, + }, + { + name: "secret being deleted - terminating", + obj: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + UID: "test-uid", + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{"test-finalizer"}, + }, + }, + rl: apiv1alpha1.ManagedControlPlane, + expected: apiv1alpha1.Terminating, + }, + } -func (l listErrorClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - return errors.New("list failed") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := SecretStatus(tt.obj, tt.rl) + assert.Equal(t, tt.expected, status.Phase) + assert.Equal(t, tt.rl, status.Location) + }) + } } type deleteErrorClient struct { diff --git a/pkg/flux/testutils_test.go b/pkg/flux/testutils_test.go new file mode 100644 index 0000000..34981ad --- /dev/null +++ b/pkg/flux/testutils_test.go @@ -0,0 +1,56 @@ +// Copyright 2025. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flux + +import ( + "context" + "errors" + + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var _ ManagedCluster = &fakeCluster{} + +type fakeCluster struct { + managedCluster + fakeClient client.Client +} + +// GetClient implements [ManagedCluster]. +func (f *fakeCluster) GetClient() client.Client { + return f.fakeClient +} + +func createFakeCluster(client client.Client) ManagedCluster { + return &fakeCluster{ + fakeClient: client, + } +} + +func createFakeClient(clusterObjects []client.Object) client.Client { + scheme := runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + return fake.NewClientBuilder().WithObjects(clusterObjects...).WithScheme(scheme).Build() +} + +type listErrorClient struct { + client.Client +} + +func (l listErrorClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return errors.New("list failed") +} diff --git a/test/e2e/flux_test.go b/test/e2e/flux_test.go index 16bd5ab..a56f310 100644 --- a/test/e2e/flux_test.go +++ b/test/e2e/flux_test.go @@ -3,9 +3,11 @@ package e2e import ( "context" "encoding/json" + "strings" "testing" "time" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -28,7 +30,16 @@ import ( "github.com/openmcp-project/openmcp-testing/pkg/resources" ) -const mcpName = "test-mcp" +const ( + mcpName = "test-mcp" + mcpCAConfigMapName = "custom-ca-bundle" + caConfigMapNameUpdate = "flux-ca-bundle-update" + caConfigMapKey = "ca.crt" + caVolumeName = "custom-ca-bundle" + caMountPath = "/etc/open-control-plane/custom-ca" + sslCertDirEnvName = "SSL_CERT_DIR" + sslCertDirEnvValue = "/etc/ssl/certs:/etc/pki/tls/certs:/etc/open-control-plane/custom-ca" +) func TestServiceProvider(t *testing.T) { var onboardingList unstructured.UnstructuredList @@ -106,6 +117,16 @@ func TestServiceProvider(t *testing.T) { if err := wait.For(conditions.New(mcp.Client().Resources()).ResourcesFound(list), wait.WithTimeout(2*time.Minute)); err != nil { t.Errorf("image pull secret not found on control plane: %v", err) } + + caBundleConfigMap := &corev1.ConfigMap{} + caBundleConfigMap.SetName(mcpCAConfigMapName) + caBundleConfigMap.SetNamespace("flux-system") + cmList := &corev1.ConfigMapList{ + Items: []corev1.ConfigMap{*caBundleConfigMap}, + } + if err := wait.For(conditions.New(mcp.Client().Resources()).ResourcesFound(cmList), wait.WithTimeout(2*time.Minute)); err != nil { + t.Errorf("ca configmap not found on control plane: %v", err) + } return ctx }). Assess("domain objects can be created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { @@ -141,6 +162,69 @@ func TestServiceProvider(t *testing.T) { } return ctx }). + Assess("flux deployments mount custom ca and set SSL_CERT_DIR", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + mcp, err := clusterutils.MCPConfig(ctx, c, mcpName) + if err != nil { + t.Error(err) + return ctx + } + + deploymentNames := []string{ + "helm-controller", + "image-automation-controller", + "image-reflector-controller", + "kustomize-controller", + "notification-controller", + "source-controller", + "source-watcher", + } + + for _, deploymentName := range deploymentNames { + deployment := &appsv1.Deployment{} + if err := mcp.Client().Resources().Get(ctx, deploymentName, "flux-system", deployment); err != nil { + t.Errorf("failed to get deployment %s: %v", deploymentName, err) + continue + } + + hasCAVolume := false + for _, volume := range deployment.Spec.Template.Spec.Volumes { + if volume.Name == caVolumeName { + hasCAVolume = true + break + } + } + if !hasCAVolume { + t.Errorf("deployment %s does not have %s volume", deploymentName, caVolumeName) + continue + } + + hasCAMount := false + hasSSLCertDirEnv := false + for _, container := range deployment.Spec.Template.Spec.Containers { + for _, volumeMount := range container.VolumeMounts { + if volumeMount.Name == caVolumeName && volumeMount.MountPath == caMountPath { + hasCAMount = true + break + } + } + for _, envVar := range container.Env { + if envVar.Name == sslCertDirEnvName && envVar.Value == sslCertDirEnvValue { + hasSSLCertDirEnv = true + break + } + } + } + + if !hasCAMount { + t.Errorf("deployment %s does not mount %s at %s", deploymentName, caVolumeName, caMountPath) + } + if !hasSSLCertDirEnv { + t.Errorf("deployment %s does not set %s=%s", deploymentName, sslCertDirEnvName, sslCertDirEnvValue) + } + } + + return ctx + }). Assess("provider config update with new secret references", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { if err := v1alpha1.AddToScheme(c.Client().Resources().GetScheme()); err != nil { t.Errorf("failed to add api types to client scheme: %s", err) @@ -225,6 +309,77 @@ func TestServiceProvider(t *testing.T) { } return ctx }). + Assess("provider config update with new ca bundle reference", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + if err := v1alpha1.AddToScheme(c.Client().Resources().GetScheme()); err != nil { + t.Errorf("failed to add api types to client scheme: %s", err) + return ctx + } + providerConfig := &v1alpha1.ProviderConfig{} + providerConfig.SetName("flux") + if err := c.Client().Resources().Get(ctx, "flux", "openmcp-system", providerConfig); err != nil { + t.Errorf("failed to get provider config: %v", err) + return ctx + } + providerConfig.Spec.CABundleRef = &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: caConfigMapNameUpdate}, + Key: caConfigMapKey, + } + if err := c.Client().Resources().Update(ctx, providerConfig); err != nil { + t.Errorf("failed to update provider config: %v", err) + return ctx + } + + onboardingConfig, err := clusterutils.OnboardingConfig() + v1alpha1.AddToScheme(onboardingConfig.GetClient().Resources().GetScheme()) + if err != nil { + t.Error(err) + return ctx + } + + fluxObj := &v1alpha1.Flux{} + fluxObj.SetName(mcpName) + fluxObj.SetNamespace(corev1.NamespaceDefault) + if err := wait.For(openmcpconditions.Match(fluxObj, onboardingConfig, "Ready", corev1.ConditionTrue), wait.WithTimeout(2*time.Minute)); err != nil { + t.Errorf("Flux not ready after provider config update: %v", err) + } + return ctx + }). + Assess("control plane ca configmap updated", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + mcp, err := clusterutils.MCPConfig(ctx, c, mcpName) + if err != nil { + t.Error(err) + return ctx + } + + // Verify that updated configmap exists + mcpCaConfigMap := &corev1.ConfigMap{} + mcpCaConfigMap.SetName(mcpCAConfigMapName) + mcpCaConfigMap.SetNamespace("flux-system") + list := &corev1.ConfigMapList{ + Items: []corev1.ConfigMap{*mcpCaConfigMap}, + } + + if err := wait.For(conditions.New(mcp.Client().Resources()).ResourcesFound(list), wait.WithTimeout(2*time.Minute)); err != nil { + t.Errorf("ca configmap not found on control plane: %v", err) + return ctx + } + + // Verify the configmap contains updated certificate data + if err := mcp.Client().Resources().Get(ctx, mcpCAConfigMapName, "flux-system", mcpCaConfigMap); err != nil { + t.Errorf("failed to get ca configmap data: %v", err) + return ctx + } + caData, ok := mcpCaConfigMap.Data[caConfigMapKey] + if !ok { + t.Errorf("ca configmap missing key %s", caConfigMapKey) + return ctx + } + // Verify the data contains the expected updated certificate marker + if !strings.Contains(caData, "UpdatedDummyCertificate") { + t.Errorf("ca configmap does not contain expected updated certificate data. Got: %s", caData) + } + return ctx + }). Assess("provider config update drops pull secrets", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { if err := v1alpha1.AddToScheme(c.Client().Resources().GetScheme()); err != nil { t.Errorf("failed to add api types to client scheme: %s", err) @@ -238,6 +393,7 @@ func TestServiceProvider(t *testing.T) { } providerConfig.Spec.Versions[0].ChartPullSecret = "" providerConfig.Spec.Versions[0].Values = nil + providerConfig.Spec.CABundleRef = nil if err := c.Client().Resources().Update(ctx, providerConfig); err != nil { t.Errorf("failed to update provider config: %v", err) } @@ -287,6 +443,21 @@ func TestServiceProvider(t *testing.T) { } return ctx }). + Assess("control plane ca configmap deleted", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + mcp, err := clusterutils.MCPConfig(ctx, c, mcpName) + if err != nil { + t.Error(err) + return ctx + } + caConfigMap := &corev1.ConfigMapList{} + if err := wait.For(conditions.New(mcp.Client().Resources().WithNamespace("flux-system")). + ResourceListN(caConfigMap, 0, klientresources.WithLabelSelector( + labels.FormatLabels(map[string]string{flux.LabelManagedBy: "service-provider-flux"}))), + wait.WithTimeout(2*time.Minute)); err != nil { + t.Errorf("orphaned ca configmap is not deleted: %v", err) + } + return ctx + }). Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { mcp, err := clusterutils.MCPConfig(ctx, c, mcpName) if err != nil { diff --git a/test/e2e/platform/caconfigmap.yaml b/test/e2e/platform/caconfigmap.yaml new file mode 100644 index 0000000..26b57c3 --- /dev/null +++ b/test/e2e/platform/caconfigmap.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flux-ca-bundle + namespace: openmcp-system +data: + # dummy value for testing + ca.crt: | + -----BEGIN CERTIFICATE----- + MIIBsjCCAVigAwIBAgIUDummyCertificateForE2ETestsOnlywCgYIKoZIzj0EAwIw + EzERMA8GA1UEAwwIdGVzdC1jYTAeFw0yNjAxMDEwMDAwMDBaFw0zNjAxMDEwMDAw + MDBaMBMxETAPBgNVBAMMCHRlc3QtY2EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC + AASxDummyCertDataOnlyForE2EFlowValidationDoNotUseInProd2Yxk7T7x7C1q5 + i8E2zX2S9qQwCgYIKoZIzj0EAwIDSAAwRQIhANdummyW9J6g4QYjv2Q6u8X9q8lQ + 3S7V8E5U2bY6uY6QhAiBiDummylX8Yv3n9vQj8m8xjz0H2xB3g9C2u8vA== + -----END CERTIFICATE----- +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: flux-ca-bundle-update + namespace: openmcp-system +data: + # dummy value for testing + ca.crt: | + -----BEGIN CERTIFICATE----- + MIIBsjCCAVigAwIBAgIUUpdatedDummyCertificateForE2ETestsOnlywCgYIKoZIzj0EAwIw + EzERMA8GA1UEAwwIdGVzdC1jYTAeFw0yNjAxMDEwMDAwMDBaFw0zNjAxMDEwMDAw + MDBaMBMxETAPBgNVBAMMCHRlc3QtY2EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC + AASxDummyCertDataOnlyForE2EFlowValidationDoNotUseInProd2Yxk7T7x7C1q5 + i8E2zX2S9qQwCgYIKoZIzj0EAwIDSAAwRQIhANdummyW9J6g4QYjv2Q6u8X9q8lQ + 3S7V8E5U2bY6uY6QhAiBiDummylX8Yv3n9vQj8m8xjz0H2xB3g9C2u8vA== + -----END CERTIFICATE----- diff --git a/test/e2e/platform/providerconfig.yaml b/test/e2e/platform/providerconfig.yaml index 384490c..e606ec4 100644 --- a/test/e2e/platform/providerconfig.yaml +++ b/test/e2e/platform/providerconfig.yaml @@ -4,6 +4,9 @@ metadata: name: flux spec: pollInterval: "10s" + caBundleRef: + name: flux-ca-bundle + key: ca.crt versions: - version: "2.8.3" chartVersion: "2.18.2"