diff --git a/api/v1alpha1/dataprotectionapplication_types.go b/api/v1alpha1/dataprotectionapplication_types.go index 1bb72ae9da1..5e9835b00ca 100644 --- a/api/v1alpha1/dataprotectionapplication_types.go +++ b/api/v1alpha1/dataprotectionapplication_types.go @@ -337,6 +337,9 @@ type VeleroConfig struct { // Velero args are settings to customize velero server arguments. Overrides values in other fields. // +optional Args *VeleroServerArgs `json:"args,omitempty"` + // LoadAffinityConfig is the config for data path load affinity. + // +optional + LoadAffinityConfig []*LoadAffinity `json:"loadAffinity,omitempty"` } // PodConfig defines the pod configuration options diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 155f3059de2..20b71b4b816 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1103,6 +1103,17 @@ func (in *VeleroConfig) DeepCopyInto(out *VeleroConfig) { *out = new(VeleroServerArgs) (*in).DeepCopyInto(*out) } + if in.LoadAffinityConfig != nil { + in, out := &in.LoadAffinityConfig, &out.LoadAffinityConfig + *out = make([]*LoadAffinity, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LoadAffinity) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VeleroConfig. diff --git a/bundle/manifests/oadp.openshift.io_dataprotectionapplications.yaml b/bundle/manifests/oadp.openshift.io_dataprotectionapplications.yaml index 96555c79b67..cddb7989c83 100644 --- a/bundle/manifests/oadp.openshift.io_dataprotectionapplications.yaml +++ b/bundle/manifests/oadp.openshift.io_dataprotectionapplications.yaml @@ -1079,6 +1079,59 @@ spec: itemOperationSyncFrequency: description: How often to check status on async backup/restore operations after backup processing. Default value is 2m. type: string + loadAffinity: + description: LoadAffinityConfig is the config for data path load affinity. + items: + description: |- + LoadAffinity is the config for data path load affinity. + Used by the Node-Agent, that needs to match the DataMover and the RepositoryMaintenance pods. + properties: + nodeSelector: + description: NodeSelector specifies the label selector to match nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array logLevel: description: Velero server's log level (use debug for the most logging, leave unset for velero default) enum: diff --git a/config/crd/bases/oadp.openshift.io_dataprotectionapplications.yaml b/config/crd/bases/oadp.openshift.io_dataprotectionapplications.yaml index bfd8d2ead01..86dfead18f0 100644 --- a/config/crd/bases/oadp.openshift.io_dataprotectionapplications.yaml +++ b/config/crd/bases/oadp.openshift.io_dataprotectionapplications.yaml @@ -1079,6 +1079,59 @@ spec: itemOperationSyncFrequency: description: How often to check status on async backup/restore operations after backup processing. Default value is 2m. type: string + loadAffinity: + description: LoadAffinityConfig is the config for data path load affinity. + items: + description: |- + LoadAffinity is the config for data path load affinity. + Used by the Node-Agent, that needs to match the DataMover and the RepositoryMaintenance pods. + properties: + nodeSelector: + description: NodeSelector specifies the label selector to match nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array logLevel: description: Velero server's log level (use debug for the most logging, leave unset for velero default) enum: diff --git a/docs/design/DPA-and-node-selectors.md b/docs/design/DPA-and-node-selectors.md index 369fe6d11e7..40411aec57c 100644 --- a/docs/design/DPA-and-node-selectors.md +++ b/docs/design/DPA-and-node-selectors.md @@ -21,11 +21,12 @@ This design enhances flexibility by allowing Affinity and Anti-Affinity settings - Guarantee that DataMover pods are scheduled on the same nodes as NodeAgent pods. - Ensure backward compatibility with existing configurations. - Ensure user experience is not degraded by the changes by providing Reconcile warnings and errors when the existing node selector configurations are used and may cause issues with the new Affinity and Anti-Affinity settings. +- Extend current Velero specific `spec.configuration.velero.podConfig` with the `spec.configuration.velero.loadAffinity` DPA CRD. The current nodeSelector from the `spec.configuration.velero.podConfig` will be kept as is. + The new `spec.configuration.velero.loadAffinity` setting will be used to configure the Velero pod affinity settings and applied to the `veleroDeployment.Spec.Template.Spec.Affinity`. ## Non Goals - Removing or deprecating existing node selector configurations. -- Use advanced Affinity settings that are not specified in teh Veleros' loadAffinity CRD such as `requiredDuringSchedulingIgnoredDuringExecution` and `preferredDuringSchedulingIgnoredDuringExecution`. -- Extend current Velero specific `spec.configuration.velero.podConfig` with the `spec.configuration.velero.loadAffinity` DPA CRD. The current nodeSelector from the `spec.configuration.velero.podConfig` will be used to schedule the Velero pod. +- Use advanced Affinity settings that are not specified in the Veleros' loadAffinity CRD such as `requiredDuringSchedulingIgnoredDuringExecution` and `preferredDuringSchedulingIgnoredDuringExecution`. ## High-Level Design The proposed change introduces a unified nodeSelector structure for all relevant components within the DPA custom resource. @@ -229,7 +230,10 @@ The `spec.configuration.velero.loadAffinity` section will be used to schedule th The use of `spec.configuration.velero.loadAffinity` will **NOT** be validated against the `spec.configuration.velero.podConfig.nodeSelector` field. This is not required restriction, because the `spec.configuration.velero.loadAffinity` is not used only to schedule the Velero pod and no dependent workloads are scheduled using the **ConfigMap** mechanism. ### Velero `spec.configuration.velero.loadAffinity` field -The `spec.configuration.velero.loadAffinity` will be added to the DPA CRD. +The `spec.configuration.velero.loadAffinity` field will be added to the DPA CRD to allow configuring affinity and anti-affinity settings for the Velero pod. This configuration will be applied to `veleroDeployment.Spec.Template.Spec.Affinity`. + +Velero pod scheduling can also be influenced by the `spec.configuration.velero.podConfig.nodeSelector` field. When both fields are used, the Velero pod will be scheduled according to standard OpenShift behavior, which supports combined use of **affinity** and **nodeSelector** to control Velero pod placement. + The updated schema will be as follows: ```yaml diff --git a/internal/controller/velero.go b/internal/controller/velero.go index be41b7889c8..9be81223ce8 100644 --- a/internal/controller/velero.go +++ b/internal/controller/velero.go @@ -13,6 +13,7 @@ import ( "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/install" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/kube" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -253,6 +254,17 @@ func (r *DataProtectionApplicationReconciler) customizeVeleroDeployment(veleroDe veleroDeployment.Spec.Template.Spec.NodeSelector = dpa.Spec.Configuration.Velero.PodConfig.NodeSelector } } + + if dpa.Spec.Configuration.Velero.LoadAffinityConfig != nil { + veleroAffinityStruct := make([]*kube.LoadAffinity, len(dpa.Spec.Configuration.Velero.LoadAffinityConfig)) + + for i, aff := range dpa.Spec.Configuration.Velero.LoadAffinityConfig { + veleroAffinityStruct[i] = (*kube.LoadAffinity)(aff) + } + affinity := kube.ToSystemAffinity(veleroAffinityStruct) + veleroDeployment.Spec.Template.Spec.Affinity = affinity + } + veleroDeployment.Spec.Template.Spec.Volumes = append(veleroDeployment.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "certs", diff --git a/internal/controller/velero_test.go b/internal/controller/velero_test.go index 5e662e7aa1a..d5910461fcd 100644 --- a/internal/controller/velero_test.go +++ b/internal/controller/velero_test.go @@ -370,6 +370,7 @@ type TestBuiltVeleroDeploymentOptions struct { resourceRequests corev1.ResourceList toleration []corev1.Toleration nodeSelector map[string]string + loadAffinity []corev1.NodeSelectorTerm } func createTestBuiltVeleroDeployment(options TestBuiltVeleroDeploymentOptions) *appsv1.Deployment { @@ -481,6 +482,16 @@ func createTestBuiltVeleroDeployment(options TestBuiltVeleroDeploymentOptions) * testBuiltVeleroDeployment.Spec.Template.Spec.NodeSelector = options.nodeSelector } + if options.loadAffinity != nil { + testBuiltVeleroDeployment.Spec.Template.Spec.Affinity = &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: options.loadAffinity, + }, + }, + } + } + if options.metricsPort != 0 { testBuiltVeleroDeployment.Spec.Template.Annotations = common.AppendTTMapAsCopy(testBuiltVeleroDeployment.Spec.Template.Annotations, map[string]string{"prometheus.io/port": strconv.Itoa(options.metricsPort)}) testBuiltVeleroDeployment.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = int32(options.metricsPort) @@ -1410,6 +1421,176 @@ func TestDPAReconciler_buildVeleroDeployment(t *testing.T) { }, }), }, + { + name: "valid DPA CR with Velero nodeselector and loadAffinity, Velero Deployment is built with nodeselector and loadAffinity", + dpa: createTestDpaWith( + nil, + oadpv1alpha1.DataProtectionApplicationSpec{ + Configuration: &oadpv1alpha1.ApplicationConfig{ + Velero: &oadpv1alpha1.VeleroConfig{ + PodConfig: &oadpv1alpha1.PodConfig{ + NodeSelector: map[string]string{"foo": "bar"}, + }, + LoadAffinityConfig: []*oadpv1alpha1.LoadAffinity{ + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"foos": "bars"}, + }, + }, + }, + }, + }, + }, + ), + veleroDeployment: testVeleroDeployment.DeepCopy(), + wantVeleroDeployment: createTestBuiltVeleroDeployment(TestBuiltVeleroDeploymentOptions{ + nodeSelector: map[string]string{"foo": "bar"}, + loadAffinity: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "foos", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"bars"}, + }, + }, + }, + }, + args: []string{ + defaultFileSystemBackupTimeout, + defaultRestoreResourcePriorities, + defaultDisableInformerCache, + }, + }), + }, + { + name: "valid DPA CR with Velero loadAffinity, Velero Deployment is built with loadAffinity", + dpa: createTestDpaWith( + nil, + oadpv1alpha1.DataProtectionApplicationSpec{ + Configuration: &oadpv1alpha1.ApplicationConfig{ + Velero: &oadpv1alpha1.VeleroConfig{ + LoadAffinityConfig: []*oadpv1alpha1.LoadAffinity{ + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"foos": "bars"}, + }, + }, + }, + }, + }, + }, + ), + veleroDeployment: testVeleroDeployment.DeepCopy(), + wantVeleroDeployment: createTestBuiltVeleroDeployment(TestBuiltVeleroDeploymentOptions{ + loadAffinity: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "foos", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"bars"}, + }, + }, + }, + }, + args: []string{ + defaultFileSystemBackupTimeout, + defaultRestoreResourcePriorities, + defaultDisableInformerCache, + }, + }), + }, + { + name: "valid DPA CR with complex Velero loadAffinity, Velero Deployment is built with loadAffinity", + dpa: createTestDpaWith( + nil, + oadpv1alpha1.DataProtectionApplicationSpec{ + Configuration: &oadpv1alpha1.ApplicationConfig{ + Velero: &oadpv1alpha1.VeleroConfig{ + LoadAffinityConfig: []*oadpv1alpha1.LoadAffinity{ + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "zone": "east", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "disk-type", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"ssd", "nvme"}, + }, + { + Key: "gpu", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + }, + { + NodeSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "instance-type", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"m5.large", "m5.xlarge"}, + }, + { + Key: "environment", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"dev"}, + }, + }, + }, + }, + }, + }, + }, + }, + ), + veleroDeployment: testVeleroDeployment.DeepCopy(), + wantVeleroDeployment: createTestBuiltVeleroDeployment(TestBuiltVeleroDeploymentOptions{ + loadAffinity: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "zone", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"east"}, + }, + { + Key: "disk-type", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"ssd", "nvme"}, + }, + { + Key: "gpu", + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + }, + }, + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "instance-type", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"m5.large", "m5.xlarge"}, + }, + { + Key: "environment", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"dev"}, + }, + }, + }, + }, + args: []string{ + defaultFileSystemBackupTimeout, + defaultRestoreResourcePriorities, + defaultDisableInformerCache, + }, + }), + }, { name: "valid DPA CR with aws plugin, Velero Deployment is built with aws plugin", dpa: createTestDpaWith(