Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ var (
}
)

func ReconcileDeployment(deployment *appsv1.Deployment, image, hcpName, openShiftVersion, kubeVersion string, ownerRef config.OwnerRef, deploymentConfig *config.DeploymentConfig, availabilityProberImage string, enableCIDebugOutput bool, platformType hyperv1.PlatformType, apiInternalPort *int32, konnectivityAddress string, konnectivityPort int32, oauthAddress string, oauthPort int32, releaseImage string, additionalTrustBundle *corev1.LocalObjectReference, hcp *hyperv1.HostedControlPlane) error {
func ReconcileDeployment(deployment *appsv1.Deployment, image, hcpName, openShiftVersion, kubeVersion string, ownerRef config.OwnerRef, deploymentConfig *config.DeploymentConfig, availabilityProberImage string, enableCIDebugOutput bool, platformType hyperv1.PlatformType, apiInternalPort *int32, konnectivityAddress string, konnectivityPort int32, oauthAddress string, oauthPort int32, releaseImage string, additionalTrustBundle *corev1.LocalObjectReference, hcp *hyperv1.HostedControlPlane, openShiftTrustedCABundleConfigMapForCPOExists bool, registryOverrides map[string]string, openShiftImageRegistryOverrides map[string][]string) error {
// Before this change we did
// Selector: &metav1.LabelSelector{
// MatchLabels: hccLabels,
Expand Down Expand Up @@ -264,7 +264,7 @@ func ReconcileDeployment(deployment *appsv1.Deployment, image, hcpName, openShif
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
util.BuildContainer(hccContainerMain(), buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion, enableCIDebugOutput, platformType, konnectivityAddress, konnectivityPort, oauthAddress, oauthPort, releaseImage)),
util.BuildContainer(hccContainerMain(), buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion, enableCIDebugOutput, platformType, konnectivityAddress, konnectivityPort, oauthAddress, oauthPort, releaseImage, registryOverrides, openShiftImageRegistryOverrides)),
},
Volumes: []corev1.Volume{
util.BuildVolume(hccVolumeKubeconfig(), buildHCCVolumeKubeconfig),
Expand All @@ -278,6 +278,9 @@ func ReconcileDeployment(deployment *appsv1.Deployment, image, hcpName, openShif
if additionalTrustBundle != nil {
util.DeploymentAddTrustBundleVolume(additionalTrustBundle, deployment)
}
if openShiftTrustedCABundleConfigMapForCPOExists {
util.DeploymentAddOpenShiftTrustedCABundleConfigMap(deployment)
}
if isExternalInfraKv(hcp) {
// injects the kubevirt credentials secret volume, volume mount path, and appends cli arg.
util.DeploymentAddKubevirtInfraCredentials(deployment)
Expand Down Expand Up @@ -319,7 +322,7 @@ func hccVolumeClusterSignerCA() *corev1.Volume {
}
}

func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string, enableCIDebugOutput bool, platformType hyperv1.PlatformType, konnectivityAddress string, konnectivityPort int32, oauthAddress string, oauthPort int32, releaseImage string) func(c *corev1.Container) {
func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string, enableCIDebugOutput bool, platformType hyperv1.PlatformType, konnectivityAddress string, konnectivityPort int32, oauthAddress string, oauthPort int32, releaseImage string, registryOverrides map[string]string, openShiftImageRegistryOverrides map[string][]string) func(c *corev1.Container) {
return func(c *corev1.Container) {
c.Image = image
c.ImagePullPolicy = corev1.PullIfNotPresent
Expand All @@ -337,6 +340,7 @@ func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string,
fmt.Sprintf("--konnectivity-port=%d", konnectivityPort),
fmt.Sprintf("--oauth-address=%s", oauthAddress),
fmt.Sprintf("--oauth-port=%d", oauthPort),
"--registry-overrides", util.ConvertRegistryOverridesToCommandLineFlag(registryOverrides),
}
if platformType == hyperv1.IBMCloudPlatform {
c.Command = append(c.Command, "--controllers=controller-manager-ca,resources,inplaceupgrader,drainer,hcpstatus")
Expand All @@ -363,6 +367,10 @@ func buildHCCContainerMain(image, hcpName, openShiftVersion, kubeVersion string,
Name: "OPERATE_ON_RELEASE_IMAGE",
Value: releaseImage,
},
{
Name: "OPENSHIFT_IMG_OVERRIDES",
Value: util.ConvertOpenShiftImageRegistryOverridesToCommandLineFlag(openShiftImageRegistryOverrides),
},
}
proxy.SetEnvVars(&c.Env)
c.VolumeMounts = volumeMounts.ContainerMounts(c.Name)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -986,6 +986,11 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont
}
}

openShiftTrustedCABundleConfigMapForCPOExists, err := doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx, r.Client, hostedControlPlane.Namespace)
if err != nil {
return err
}

r.Log.Info("Reconciling ignition server")
if err := ignitionserver.ReconcileIgnitionServer(ctx,
r.Client,
Expand All @@ -1002,6 +1007,8 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont
util.ConvertOpenShiftImageRegistryOverridesToCommandLineFlag(r.ReleaseProvider.GetOpenShiftImageRegistryOverrides()),
r.ManagementClusterCapabilities.Has(capabilities.CapabilitySecurityContextConstraint),
config.OwnerRefFrom(hostedControlPlane),
openShiftTrustedCABundleConfigMapForCPOExists,
r.ReleaseProvider.GetMirroredReleaseImage(),
); err != nil {
return fmt.Errorf("failed to reconcile ignition server: %w", err)
}
Expand Down Expand Up @@ -1070,7 +1077,7 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont

// Reconcile hosted cluster config operator
r.Log.Info("Reconciling Hosted Cluster Config Operator")
if err := r.reconcileHostedClusterConfigOperator(ctx, hostedControlPlane, userReleaseImageProvider, infraStatus, createOrUpdate); err != nil {
if err := r.reconcileHostedClusterConfigOperator(ctx, hostedControlPlane, userReleaseImageProvider, infraStatus, createOrUpdate, openShiftTrustedCABundleConfigMapForCPOExists); err != nil {
return fmt.Errorf("failed to reconcile hosted cluster config operator: %w", err)
}

Expand Down Expand Up @@ -3441,7 +3448,7 @@ func removeServiceCAAnnotationAndSecret(ctx context.Context, c client.Client, se
return nil
}

func (r *HostedControlPlaneReconciler) reconcileHostedClusterConfigOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider *imageprovider.ReleaseImageProvider, infraStatus InfrastructureStatus, createOrUpdate upsert.CreateOrUpdateFN) error {
func (r *HostedControlPlaneReconciler) reconcileHostedClusterConfigOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider *imageprovider.ReleaseImageProvider, infraStatus InfrastructureStatus, createOrUpdate upsert.CreateOrUpdateFN, openShiftTrustedCABundleConfigMapForCPOExists bool) error {
versions, err := releaseImageProvider.ComponentVersions()
if err != nil {
return fmt.Errorf("failed to get component versions: %w", err)
Expand Down Expand Up @@ -3471,7 +3478,7 @@ func (r *HostedControlPlaneReconciler) reconcileHostedClusterConfigOperator(ctx

deployment := manifests.ConfigOperatorDeployment(hcp.Namespace)
if _, err = createOrUpdate(ctx, r.Client, deployment, func() error {
return configoperator.ReconcileDeployment(deployment, p.Image, hcp.Name, p.OpenShiftVersion, p.KubernetesVersion, p.OwnerRef, &p.DeploymentConfig, p.AvailabilityProberImage, r.EnableCIDebugOutput, hcp.Spec.Platform.Type, util.APIPort(hcp), infraStatus.KonnectivityHost, infraStatus.KonnectivityPort, infraStatus.OAuthHost, infraStatus.OAuthPort, hcp.Spec.ReleaseImage, hcp.Spec.AdditionalTrustBundle, hcp)
return configoperator.ReconcileDeployment(deployment, p.Image, hcp.Name, p.OpenShiftVersion, p.KubernetesVersion, p.OwnerRef, &p.DeploymentConfig, p.AvailabilityProberImage, r.EnableCIDebugOutput, hcp.Spec.Platform.Type, util.APIPort(hcp), infraStatus.KonnectivityHost, infraStatus.KonnectivityPort, infraStatus.OAuthHost, infraStatus.OAuthPort, hcp.Spec.ReleaseImage, hcp.Spec.AdditionalTrustBundle, hcp, openShiftTrustedCABundleConfigMapForCPOExists, r.ReleaseProvider.GetRegistryOverrides(), r.ReleaseProvider.GetOpenShiftImageRegistryOverrides())
}); err != nil {
return fmt.Errorf("failed to reconcile config operator deployment: %w", err)
}
Expand Down Expand Up @@ -4299,3 +4306,17 @@ func (r *HostedControlPlaneReconciler) reconcileSREMetricsConfig(ctx context.Con
}
return nil
}

func doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx context.Context, c client.Client, hcpNamespace string) (bool, error) {
openShiftTrustedCABundleConfigMapForCPO := manifests.OpenShiftTrustedCABundleFromCPO(hcpNamespace)
if err := c.Get(ctx, client.ObjectKeyFromObject(openShiftTrustedCABundleConfigMapForCPO), openShiftTrustedCABundleConfigMapForCPO); err != nil {
// It's okay if this ConfigMap doesn't exist. It won't for non-OCP clusters. Only return an error if the error is something other than not existing.
if !apierrors.IsNotFound(err) {
return false, fmt.Errorf("error getting %T: %w", openShiftTrustedCABundleConfigMapForCPO, err)
}
}
if openShiftTrustedCABundleConfigMapForCPO.Data != nil {
return true, nil
}
return false, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ func ReconcileIgnitionServer(ctx context.Context,
openShiftRegistryOverrides string,
managementClusterHasCapabilitySecurityContextConstraint bool,
ownerRef config.OwnerRef,
openShiftTrustedCABundleConfigMapExists bool,
mirroredReleaseImage string,
) error {
log := ctrl.LoggerFrom(ctx)

Expand Down Expand Up @@ -234,7 +236,10 @@ func ReconcileIgnitionServer(ctx context.Context,
openShiftRegistryOverrides,
managementClusterHasCapabilitySecurityContextConstraint,
ignitionServerLabels,
servingCertSecretName)
servingCertSecretName,
openShiftTrustedCABundleConfigMapExists,
mirroredReleaseImage,
)
}); err != nil {
return fmt.Errorf("failed to reconcile ignition deployment: %w", err)
} else {
Expand Down Expand Up @@ -467,6 +472,8 @@ func reconcileDeployment(deployment *appsv1.Deployment,
managementClusterHasCapabilitySecurityContextConstraint bool,
ignitionServerLabels map[string]string,
servingCertSecretName string,
openShiftTrustedCABundleConfigMapForCPOExists bool,
mirroredReleaseImage string,
) error {
var probeHandler corev1.ProbeHandler
if hasHealthzHandler {
Expand Down Expand Up @@ -693,11 +700,19 @@ func reconcileDeployment(deployment *appsv1.Deployment,
}
proxy.SetEnvVars(&deployment.Spec.Template.Spec.Containers[0].Env)

if len(mirroredReleaseImage) > 0 {
deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "MIRRORED_RELEASE_IMAGE", Value: mirroredReleaseImage})
}

if hcp.Spec.AdditionalTrustBundle != nil {
// Add trusted-ca mount with optional configmap
util.DeploymentAddTrustBundleVolume(hcp.Spec.AdditionalTrustBundle, deployment)
}

if openShiftTrustedCABundleConfigMapForCPOExists {
util.DeploymentAddOpenShiftTrustedCABundleConfigMap(deployment)
}

// set security context
if !managementClusterHasCapabilitySecurityContextConstraint {
deployment.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package manifests

import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func OpenShiftTrustedCABundleFromCPO(namespace string) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: "openshift-config-managed-trusted-ca-bundle",
},
}
}
36 changes: 27 additions & 9 deletions control-plane-operator/hostedclusterconfigoperator/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import (
"fmt"
"os"

"k8s.io/client-go/rest"

hyperv1 "github.com/openshift/hypershift/api/v1beta1"
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/api"
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/configmetrics"
Expand All @@ -34,8 +32,12 @@ import (
"github.com/openshift/hypershift/support/labelenforcingclient"
"github.com/openshift/hypershift/support/releaseinfo"
"github.com/openshift/hypershift/support/upsert"
"github.com/openshift/hypershift/support/util"

"github.com/spf13/cobra"
"go.uber.org/zap/zapcore"

"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
Expand Down Expand Up @@ -107,6 +109,8 @@ type HostedClusterConfigOperator struct {
enableCIDebugOutput bool

clusterSignerCA []byte

registryOverrides map[string]string
}

func newHostedClusterConfigOperatorCommand() *cobra.Command {
Expand Down Expand Up @@ -139,6 +143,7 @@ func newHostedClusterConfigOperatorCommand() *cobra.Command {
flags.Int32Var(&cpo.KonnectivityPort, "konnectivity-port", cpo.KonnectivityPort, "Port of external konnectivity endpoint")
flags.StringVar(&cpo.OAuthAddress, "oauth-address", cpo.KonnectivityAddress, "Address of external oauth endpoint")
flags.Int32Var(&cpo.OAuthPort, "oauth-port", cpo.KonnectivityPort, "Port of external oauth endpoint")
flags.StringToStringVar(&cpo.registryOverrides, "registry-overrides", map[string]string{}, "registry-overrides contains the source registry string as a key and the destination registry string as value. Images before being applied are scanned for the source registry string and if found the string is replaced with the destination registry string. Format is: sr1=dr1,sr2=dr2")
return cmd
}

Expand Down Expand Up @@ -224,15 +229,28 @@ func (o *HostedClusterConfigOperator) Run(ctx context.Context) error {
kubevirtInfraConfig = cpConfig
}

releaseProvider := &releaseinfo.StaticProviderDecorator{
Delegate: &releaseinfo.CachedProvider{
Inner: &releaseinfo.RegistryClientProvider{},
Cache: map[string]*releaseinfo.ReleaseImage{},
},
ComponentImages: map[string]string{
"konnectivity-agent": konnectivityAgentImage,
var imageRegistryOverrides map[string][]string
openShiftImgOverrides, ok := os.LookupEnv("OPENSHIFT_IMG_OVERRIDES")
if ok {
imageRegistryOverrides = util.ConvertImageRegistryOverrideStringToMap(openShiftImgOverrides)
}

releaseProvider := &releaseinfo.ProviderWithOpenShiftImageRegistryOverridesDecorator{
Delegate: &releaseinfo.RegistryMirrorProviderDecorator{
Delegate: &releaseinfo.StaticProviderDecorator{
Delegate: &releaseinfo.CachedProvider{
Inner: &releaseinfo.RegistryClientProvider{},
Cache: map[string]*releaseinfo.ReleaseImage{},
},
ComponentImages: map[string]string{
"konnectivity-agent": konnectivityAgentImage,
},
},
RegistryOverrides: o.registryOverrides,
},
OpenShiftImageRegistryOverrides: imageRegistryOverrides,
}

operatorConfig := &operator.HostedClusterConfigOperatorConfig{
TargetCreateOrUpdateProvider: &labelenforcingclient.LabelEnforcingUpsertProvider{
Upstream: upsert.New(o.enableCIDebugOutput),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1027,7 +1027,7 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to get controlPlaneOperatorImage: %w", err)
}
controlPlaneOperatorImageMetadata, err := r.ImageMetadataProvider.ImageMetadata(ctx, controlPlaneOperatorImage, pullSecretBytes, hcluster.Spec.ImageContentSources)
controlPlaneOperatorImageMetadata, err := r.ImageMetadataProvider.ImageMetadata(ctx, controlPlaneOperatorImage, pullSecretBytes)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to look up image metadata for %s: %w", controlPlaneOperatorImage, err)
}
Expand Down Expand Up @@ -1580,8 +1580,13 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
return ctrl.Result{}, fmt.Errorf("failed to reconcile SRE metrics config: %w", err)
}

openShiftTrustedCABundleConfigMapExists, err := r.reconcileOpenShiftTrustedCAs(ctx, hcp)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to reconcile OpenShift trusted CAs: %w", err)
}

// Reconcile the control plane operator
err = r.reconcileControlPlaneOperator(ctx, createOrUpdate, hcluster, hcp, controlPlaneOperatorImage, utilitiesImage, defaultIngressDomain, cpoHasUtilities)
err = r.reconcileControlPlaneOperator(ctx, createOrUpdate, hcluster, hcp, controlPlaneOperatorImage, utilitiesImage, defaultIngressDomain, cpoHasUtilities, openShiftTrustedCABundleConfigMapExists)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to reconcile control plane operator: %w", err)
}
Expand Down Expand Up @@ -1999,7 +2004,7 @@ func (r *HostedClusterReconciler) reconcileCAPIProvider(ctx context.Context, cre

// reconcileControlPlaneOperator orchestrates reconciliation of the control plane
// operator components.
func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Context, createOrUpdate upsert.CreateOrUpdateFN, hcluster *hyperv1.HostedCluster, hostedControlPlane *hyperv1.HostedControlPlane, controlPlaneOperatorImage, utilitiesImage, defaultIngressDomain string, cpoHasUtilities bool) error {
func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Context, createOrUpdate upsert.CreateOrUpdateFN, hcluster *hyperv1.HostedCluster, hostedControlPlane *hyperv1.HostedControlPlane, controlPlaneOperatorImage, utilitiesImage, defaultIngressDomain string, cpoHasUtilities bool, openShiftTrustedCABundleConfigMapExists bool) error {
controlPlaneNamespace := manifests.HostedControlPlaneNamespace(hcluster.Namespace, hcluster.Name)
err := r.Client.Get(ctx, client.ObjectKeyFromObject(controlPlaneNamespace), controlPlaneNamespace)
if err != nil {
Expand Down Expand Up @@ -2075,6 +2080,7 @@ func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Cont
_, err = createOrUpdate(ctx, r.Client, controlPlaneOperatorDeployment, func() error {
return reconcileControlPlaneOperatorDeployment(
controlPlaneOperatorDeployment,
openShiftTrustedCABundleConfigMapExists,
hcluster,
hostedControlPlane,
controlPlaneOperatorImage,
Expand Down Expand Up @@ -2120,6 +2126,41 @@ func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Cont
return nil
}

// reconcileOpenShiftTrustedCAs checks for the existence of /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem, if it exists,
// creates a new ConfigMap to be mounted in the CPO deployment utilizing the file
func (r *HostedClusterReconciler) reconcileOpenShiftTrustedCAs(ctx context.Context, hostedControlPlane *hyperv1.HostedControlPlane) (bool, error) {
trustedCABundle := new(bytes.Buffer)
var trustCABundleFile []byte

_, err := os.Stat("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem")
if err == nil {
trustCABundleFile, err = os.ReadFile("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem")
if err != nil {
return false, fmt.Errorf("unable to read trust bundle file: %w", err)
}
}
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return false, nil
}

return false, err
}

if _, err = trustedCABundle.Write(trustCABundleFile); err != nil {
return false, fmt.Errorf("unable to write trust bundle to buffer: %w", err)
}

// Next, save the contents to a new ConfigMap in the hosted control plane's namespace
openShiftTrustedCABundleConfigMapForCPO := manifests.OpenShiftTrustedCABundleForNamespace(hostedControlPlane.Namespace)
openShiftTrustedCABundleConfigMapForCPO.Data["ca-bundle.crt"] = trustedCABundle.String()
if _, err = controllerutil.CreateOrUpdate(ctx, r.Client, openShiftTrustedCABundleConfigMapForCPO, NoopReconcile); err != nil {
return false, fmt.Errorf("failed to create openshift-config-managed-trusted-ca-bundle for CPO deployment %T: %w", trustedCABundle.String(), err)
}

return true, nil
}

func servicePublishingStrategyByType(hcp *hyperv1.HostedCluster, svcType hyperv1.ServiceType) *hyperv1.ServicePublishingStrategy {
for _, mapping := range hcp.Spec.Services {
if mapping.Service == svcType {
Expand Down Expand Up @@ -2183,6 +2224,7 @@ func GetControlPlaneOperatorImage(ctx context.Context, hc *hyperv1.HostedCluster

func reconcileControlPlaneOperatorDeployment(
deployment *appsv1.Deployment,
openShiftTrustedCABundleConfigMapExists bool,
hc *hyperv1.HostedCluster,
hcp *hyperv1.HostedControlPlane,
cpoImage,
Expand Down Expand Up @@ -2322,6 +2364,10 @@ func reconcileControlPlaneOperatorDeployment(
},
}

if openShiftTrustedCABundleConfigMapExists {
hyperutil.DeploymentAddOpenShiftTrustedCABundleConfigMap(deployment)
}

if os.Getenv(rhobsmonitoring.EnvironmentVariable) == "1" {
deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env,
corev1.EnvVar{
Expand Down
Loading