diff --git a/Makefile b/Makefile index e3e8535845..622842bed1 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,13 @@ test-e2e-encryption: GO_TEST_FLAGS += -parallel 1 test-e2e-encryption: test-unit .PHONY: test-e2e-encryption +test-e2e-encryption-perf: GO_TEST_PACKAGES :=./test/e2e-encryption-perf/... +test-e2e-encryption-perf: GO_TEST_FLAGS += -v +test-e2e-encryption-perf: GO_TEST_FLAGS += -timeout 1h +test-e2e-encryption-perf: GO_TEST_FLAGS += -p 1 +test-e2e-encryption-perf: test-unit +.PHONY: test-e2e-encryption-perf + # Configure the 'telepresence' target # See vendor/github.com/openshift/build-machinery-go/scripts/run-telepresence.sh for usage and configuration details export TP_DEPLOYMENT_YAML ?=./manifests/07_deployment.yaml diff --git a/go.mod b/go.mod index 0d1b147fe8..faf7b5b159 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/openshift/api v0.0.0-20201019163320-c6a5ec25f267 github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab github.com/openshift/client-go v0.0.0-20201020074620-f8fd44879f7c - github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08 + github.com/openshift/library-go v0.0.0-20201202092334-3557e59845d7 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index 822318158c..0086f4df09 100644 --- a/go.sum +++ b/go.sum @@ -353,8 +353,8 @@ github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab h1:lB github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/client-go v0.0.0-20201020074620-f8fd44879f7c h1:NB9g4Y/aegId7fyNqYyGxEfyNOytYFT5dxWJtfOJFQs= github.com/openshift/client-go v0.0.0-20201020074620-f8fd44879f7c/go.mod h1:yZ3u8vgWC19I9gbDMRk8//9JwG/0Sth6v7C+m6R8HXs= -github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08 h1:Z+8t3ooTH2T+J/GoCZbgaOk5WqNZgPuHlUAKMfG1FEk= -github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08/go.mod h1:1xYaYQcQsn+AyCRsvOU+Qn5z6GGiCmcblXkT/RZLVfo= +github.com/openshift/library-go v0.0.0-20201202092334-3557e59845d7 h1:haE99sHkbQs+QPIFHKgV8RKZ9EuBRFXraIwwEi0rX98= +github.com/openshift/library-go v0.0.0-20201202092334-3557e59845d7/go.mod h1:1xYaYQcQsn+AyCRsvOU+Qn5z6GGiCmcblXkT/RZLVfo= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= diff --git a/pkg/operator/encryptionprovider/encryptionprovider_test.go b/pkg/operator/encryptionprovider/encryptionprovider_test.go index b85a9fd80d..c6302817ef 100644 --- a/pkg/operator/encryptionprovider/encryptionprovider_test.go +++ b/pkg/operator/encryptionprovider/encryptionprovider_test.go @@ -30,11 +30,14 @@ func TestEncryptionProvider(t *testing.T) { expectedShouldRunEncryptionCtrl bool }{ { + // this will be default in 4.7 name: "encryption off, default GRs returned", defaultEncryptedGRs: defaultGRs, - expectedShouldRunEncryptionCtrl: false, + expectedEncryptedGRs: defaultGRs, + expectedShouldRunEncryptionCtrl: true, }, { + // it means that encryption was on in 4.7 or the cluster was updated from 4.6 and the annotation was removed name: "encryption on, secret without the annotation, default GRs returned", initialSecrets: []*corev1.Secret{ func() *corev1.Secret { @@ -48,6 +51,7 @@ func TestEncryptionProvider(t *testing.T) { expectedShouldRunEncryptionCtrl: true, }, { + // it means that encryption was on in 4.6 and migration hasn't finished yet name: "encryption on, secret with the annotation, reduced GRs returned", initialSecrets: []*corev1.Secret{defaultSecret("openshift-apiserver", encryptionCfgAnnotationKey)}, defaultEncryptedGRs: defaultGRs, diff --git a/pkg/operator/encryptionprovider/encyrptionprovider.go b/pkg/operator/encryptionprovider/encyrptionprovider.go index c9dfcce69c..72014ddb87 100644 --- a/pkg/operator/encryptionprovider/encyrptionprovider.go +++ b/pkg/operator/encryptionprovider/encyrptionprovider.go @@ -40,11 +40,16 @@ func New( // EncryptedGRs returns resources that need to be encrypted // Note: the list can change depending on the existence and attached annotations of encryption-config-openshift-oauth-apiserver in openshift-config-managed namespace as described in https://github.com/openshift/enhancements/blob/master/enhancements/etcd/etcd-encryption-for-separate-oauth-apis.md // -// case 1 encryption off or the secret was annotated - return an empty list of EncryptedGRs -// case 2 otherwise return the authoritative list of EncryptedGRs +// case 1 encryption off OR the encryption-config-openshift-oauth-apiserver doesn't have "encryption.apiserver.operator.openshift.io/managed-by" annotation +// - we will the authoritative list of EncryptedGRs +// - at the time of this writhing that list contains: +// {Group: "oauth.openshift.io", Resource: "oauthaccesstokens"} +// {Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"} // -// TODO: -// - change the code in 4.7 to return a static list (the authoritative list) +// case 2 otherwise (the encryption config doesn't have the annotation) return an empty list of EncryptedGRs - that means that OAS-O is still in charge of the encryption config +// +// TODO in 4.8 +// this encryption provided won't be needed and can be removed func (p *encryptionProvider) EncryptedGRs() []schema.GroupResource { inCharge, err := p.isOAuthEncryptionConfigManagedByThisOperator() if err != nil || !inCharge { @@ -59,12 +64,13 @@ func (p *encryptionProvider) ShouldRunEncryptionControllers() (bool, error) { return p.isOAuthEncryptionConfigManagedByThisOperator() } -// isOAuthEncryptionConfigManagedByThisOperator determines whether this operator is in charge of encryption-config-openshift-oauth-apiserver +// isOAuthEncryptionConfigManagedByThisOperator determines whether this operator is in charge of encryption-config-openshift-oauth-apiserver in openshift-config-managed namespace +// +// case 1 encryption off OR the encryption config doesn't have "encryption.apiserver.operator.openshift.io/managed-by" annotation - this operator is in charge +// case 2 the encryption config have the annotation - OAS-O is still in charge // -// case 1 encryption off or the secret was annotated - OAS-O is in charge -// case 2 otherwise this operator will manage its own encryption configuration -// TODO: -// - change the case 1 in 4.7 so that CAO manages its own encryption config when encryption is off +// TODO in 4.8 +// this encryption provider won't be needed and can be removed func (p *encryptionProvider) isOAuthEncryptionConfigManagedByThisOperator() (bool, error) { oauthAPIServerEncryptionCfgName := fmt.Sprintf("%s-%s", encryptionconfig.EncryptionConfSecretName, p.targetNamespace) oauthAPIServerEncryptionCfg, err := p.secretLister.Get(oauthAPIServerEncryptionCfgName) @@ -74,10 +80,10 @@ func (p *encryptionProvider) isOAuthEncryptionConfigManagedByThisOperator() (boo // - we suppress the error so that the encryption controllers: // 1. don't report Degraded when encryption is off // 2. don't requeue when encryption is off - return false, nil // case 1 - OAS-O in charge + return true, nil // case 1 - we are in charge } if _, exist := oauthAPIServerEncryptionCfg.Annotations[p.encryptionCfgAnnotationKey]; exist { - return false, nil // case 1 - OAS-O in charge + return false, nil // case 2 - OAS-O is in charge } - return true, nil // case 2 - taking over + return true, nil // case 1 - taking over } diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 89b7ee5587..14a2f085b0 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -548,6 +548,8 @@ func prepareOauthAPIServerOperator(ctx context.Context, controllerContext *contr operatorCtx.configClient.ConfigV1().APIServers(), operatorCtx.operatorConfigInformer.Config().V1().APIServers(), operatorCtx.kubeInformersForNamespaces, + ).WithUnsupportedConfigPrefixForEncryptionControllers( + oauthapiconfigobservercontroller.OAuthAPIServerConfigPrefix, ).WithFinalizerController( "openshift-oauth-apiserver", operatorCtx.kubeInformersForNamespaces.InformersFor("openshift-oauth-apiserver"), diff --git a/test/e2e-encryption-perf/encryption_perf_test.go b/test/e2e-encryption-perf/encryption_perf_test.go new file mode 100644 index 0000000000..68815418a9 --- /dev/null +++ b/test/e2e-encryption-perf/encryption_perf_test.go @@ -0,0 +1,105 @@ +package e2e_encryption_perf + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" + + oauthapiv1 "github.com/openshift/api/oauth/v1" + operatorv1 "github.com/openshift/api/operator/v1" + oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1" + operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library" + operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" + library "github.com/openshift/library-go/test/library/encryption" +) + +const ( + tokenStatsKey = "created oauthaccesstokens" +) + +func TestPerfEncryptionTypeAESCBC(tt *testing.T) { + ctx := context.TODO() + clientSet := getPerfClients(tt) + library.TestPerfEncryptionTypeAESCBC(tt, library.PerfScenario{ + BasicScenario: library.BasicScenario{ + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", + EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"), + EncryptionConfigSecretNamespace: "openshift-config-managed", + OperatorNamespace: "openshift-authentication-operator", + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, + }, + GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) { + apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return nil, err + } + return apiServerOperator.Status.Conditions, nil + }, + AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) { + tokenCount, ok := statStore[tokenStatsKey] + if !ok { + err := errors.New("missing oauth access tokens count stats, can't continue the test") + require.NoError(t, err) + } + if tokenCount < 14000 { + err := fmt.Errorf("expected to create at least 14000 tokens but %d were created", tokenCount) + require.NoError(t, err) + } + t.Logf("Created %d access tokens", tokenCount) + }, + AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) { + t.Logf("migration took %v", migrationTime) + expectedMigrationTime := 10 * time.Minute + if migrationTime > expectedMigrationTime { + t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime) + } + }, + DBLoaderWorkers: 3, + DBLoaderFunc: library.DBLoaderRepeat(1, false, + library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)), + }) +} + +func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType { + return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error { + token := &oauthapiv1.OAuthAccessToken{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("encryption-token-%v", rand.String(18)), + }, + RefreshToken: "I have no special talents. I am only passionately curious", + UserName: "kube:admin", + Scopes: []string{"user:full"}, + RedirectURI: "redirect.me.to.token.of.life", + ClientName: "console", + UserUID: "non-existing-user-id", + } + _, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{}) + return err + } +} + +func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error { + statsCollector(tokenStatsKey) + return nil +} + +func getPerfClients(t *testing.T) operatorencryption.ClientSet { + t.Helper() + + kubeConfig := operatorlibrary.NewClientConfigForTest(t) + + kubeConfig.QPS = 300 + kubeConfig.Burst = 600 + + return operatorencryption.GetClientsFor(t, kubeConfig) +} diff --git a/test/e2e-encryption/encryption_test.go b/test/e2e-encryption/encryption_test.go index 2d8942c2ca..36c1271115 100644 --- a/test/e2e-encryption/encryption_test.go +++ b/test/e2e-encryption/encryption_test.go @@ -2,75 +2,60 @@ package e2eencryption import ( "context" + "encoding/json" "fmt" - "strings" "testing" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - configv1 "github.com/openshift/api/config/v1" - oauthapiv1 "github.com/openshift/api/oauth/v1" + oauthapiconfigobservercontroller "github.com/openshift/cluster-authentication-operator/pkg/operator/configobservation" + operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" library "github.com/openshift/library-go/test/library/encryption" ) -var DefaultTargetGRs = []schema.GroupResource{ - {Group: "oauth.openshift.io", Resource: "oauthaccesstokens"}, - {Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"}, - // TODO: remove route in 4.7, in 4.6 OAS-O is managing the encryption configuration for CAO - {Group: "route.openshift.io", Resource: "routes"}, -} - func TestEncryptionTypeIdentity(t *testing.T) { library.TestEncryptionTypeIdentity(t, library.BasicScenario{ - Namespace: "openshift-config-managed", - // TODO: update the LabelSelector in 4.7, in 4.6 OAS-O is managing the encryption configuration for CAO - // LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver" - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-apiserver", + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), EncryptionConfigSecretNamespace: "openshift-config-managed", OperatorNamespace: "openshift-authentication-operator", - TargetGRs: DefaultTargetGRs, - AssertFunc: assertTokens, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, }) } func TestEncryptionTypeUnset(t *testing.T) { library.TestEncryptionTypeUnset(t, library.BasicScenario{ - Namespace: "openshift-config-managed", - // TODO: update the LabelSelector in 4.7, in 4.6 OAS-O is managing the encryption configuration for CAO - // LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver" - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-apiserver", + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), EncryptionConfigSecretNamespace: "openshift-config-managed", OperatorNamespace: "openshift-authentication-operator", - TargetGRs: DefaultTargetGRs, - AssertFunc: assertTokens, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, }) } func TestEncryptionTurnOnAndOff(t *testing.T) { library.TestEncryptionTurnOnAndOff(t, library.OnOffScenario{ BasicScenario: library.BasicScenario{ - Namespace: "openshift-config-managed", - // TODO: update the LabelSelector in 4.7, in 4.6 OAS-O is managing the encryption configuration for CAO - // LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver" - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-apiserver", + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), EncryptionConfigSecretNamespace: "openshift-config-managed", OperatorNamespace: "openshift-authentication-operator", - TargetGRs: DefaultTargetGRs, - AssertFunc: assertTokens, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, }, CreateResourceFunc: func(t testing.TB, _ library.ClientSet, namespace string) runtime.Object { - return CreateAndStoreTokenOfLife(context.TODO(), t, GetClients(t)) + return operatorencryption.CreateAndStoreTokenOfLife(context.TODO(), t, operatorencryption.GetClients(t)) }, - AssertResourceEncryptedFunc: assertTokenOfLifeEncrypted, - AssertResourceNotEncryptedFunc: assertTokenOfLifeNotEncrypted, - ResourceFunc: func(t testing.TB, _ string) runtime.Object { return TokenOfLife(t) }, + AssertResourceEncryptedFunc: operatorencryption.AssertTokenOfLifeEncrypted, + AssertResourceNotEncryptedFunc: operatorencryption.AssertTokenOfLifeNotEncrypted, + ResourceFunc: func(t testing.TB, _ string) runtime.Object { return operatorencryption.TokenOfLife(t) }, ResourceName: "TokenOfLife", }) } @@ -81,70 +66,48 @@ func TestEncryptionRotation(t *testing.T) { ctx := context.TODO() library.TestEncryptionRotation(t, library.RotationScenario{ BasicScenario: library.BasicScenario{ - Namespace: "openshift-config-managed", - // TODO: update the LabelSelector in 4.7, in 4.6 OAS-O is managing the encryption configuration for CAO - // LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver" - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-apiserver", + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), EncryptionConfigSecretNamespace: "openshift-config-managed", OperatorNamespace: "openshift-authentication-operator", - TargetGRs: DefaultTargetGRs, - AssertFunc: assertTokens, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, }, CreateResourceFunc: func(t testing.TB, _ library.ClientSet, _ string) runtime.Object { - return CreateAndStoreTokenOfLife(ctx, t, GetClients(t)) + return operatorencryption.CreateAndStoreTokenOfLife(ctx, t, operatorencryption.GetClients(t)) }, GetRawResourceFunc: func(t testing.TB, clientSet library.ClientSet, _ string) string { - return GetRawTokenOfLife(t, clientSet) + return operatorencryption.GetRawTokenOfLife(t, clientSet) }, - UnsupportedConfigFunc: func(raw []byte) error { - cs := GetClients(t) - apiServerOperator, err := cs.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + UnsupportedConfigFunc: func(rawUnsupportedEncryptionCfg []byte) error { + cs := operatorencryption.GetClients(t) + authOperator, err := cs.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + unsupportedConfigAsMap := map[string]interface{}{} + if len(authOperator.Spec.UnsupportedConfigOverrides.Raw) > 0 { + if err := json.Unmarshal(authOperator.Spec.UnsupportedConfigOverrides.Raw, &unsupportedConfigAsMap); err != nil { + return err + } + } + unsupportedEncryptionConfigAsMap := map[string]interface{}{} + if err := json.Unmarshal(rawUnsupportedEncryptionCfg, &unsupportedEncryptionConfigAsMap); err != nil { + return err + } + if err := unstructured.SetNestedMap(unsupportedConfigAsMap, unsupportedEncryptionConfigAsMap, oauthapiconfigobservercontroller.OAuthAPIServerConfigPrefix); err != nil { + return err + } + rawUnsupportedCfg, err := json.Marshal(unsupportedConfigAsMap) if err != nil { return err } - apiServerOperator.Spec.UnsupportedConfigOverrides.Raw = raw - _, err = cs.OperatorClient.Update(ctx, apiServerOperator, metav1.UpdateOptions{}) + authOperator.Spec.UnsupportedConfigOverrides.Raw = rawUnsupportedCfg + + _, err = cs.OperatorClient.Update(ctx, authOperator, metav1.UpdateOptions{}) return err }, }) } - -func assertTokens(t testing.TB, clientSet library.ClientSet, expectedMode configv1.EncryptionType, namespace, labelSelector string) { - t.Helper() - assertAccessTokens(t, clientSet.Etcd, string(expectedMode)) - assertAuthTokens(t, clientSet.Etcd, string(expectedMode)) - library.AssertLastMigratedKey(t, clientSet.Kube, DefaultTargetGRs, namespace, labelSelector) -} - -func assertAccessTokens(t testing.TB, etcdClient library.EtcdClient, expectedMode string) { - t.Logf("Checking if all OauthAccessTokens where encrypted/decrypted for %q mode", expectedMode) - totalAccessTokens, err := library.VerifyResources(t, etcdClient, "/openshift.io/oauth/accesstokens/", expectedMode, true) - t.Logf("Verified %d OauthAccessTokens", totalAccessTokens) - require.NoError(t, err) -} - -func assertAuthTokens(t testing.TB, etcdClient library.EtcdClient, expectedMode string) { - t.Logf("Checking if all OAuthAuthorizeTokens where encrypted/decrypted for %q mode", expectedMode) - totalAuthTokens, err := library.VerifyResources(t, etcdClient, "/openshift.io/oauth/authorizetokens/", expectedMode, true) - t.Logf("Verified %d OAuthAuthorizeTokens", totalAuthTokens) - require.NoError(t, err) -} - -func assertTokenOfLifeEncrypted(t testing.TB, clientSet library.ClientSet, rawTokenOfLife runtime.Object) { - t.Helper() - tokenOfLife := rawTokenOfLife.(*oauthapiv1.OAuthAccessToken) - rawTokenValue := GetRawTokenOfLife(t, clientSet) - if strings.Contains(rawTokenValue, tokenOfLife.RefreshToken) { - t.Errorf("access token not encrypted, token received from etcd have %q (plain text), raw content in etcd is %s", tokenOfLife.RefreshToken, rawTokenValue) - } -} - -func assertTokenOfLifeNotEncrypted(t testing.TB, clientSet library.ClientSet, rawTokenOfLife runtime.Object) { - t.Helper() - tokenOfLife := rawTokenOfLife.(*oauthapiv1.OAuthAccessToken) - rawTokenValue := GetRawTokenOfLife(t, clientSet) - if !strings.Contains(rawTokenValue, tokenOfLife.RefreshToken) { - t.Errorf("access token received from etcd doesnt have %q (plain text), raw content in etcd is %s", tokenOfLife.RefreshToken, rawTokenValue) - } -} diff --git a/test/library/encryption/assertion.go b/test/library/encryption/assertion.go new file mode 100644 index 0000000000..0d3ec3ca77 --- /dev/null +++ b/test/library/encryption/assertion.go @@ -0,0 +1,59 @@ +package encryption + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + configv1 "github.com/openshift/api/config/v1" + oauthapiv1 "github.com/openshift/api/oauth/v1" + library "github.com/openshift/library-go/test/library/encryption" +) + +var DefaultTargetGRs = []schema.GroupResource{ + {Group: "oauth.openshift.io", Resource: "oauthaccesstokens"}, + {Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"}, +} + +func AssertTokens(t testing.TB, clientSet library.ClientSet, expectedMode configv1.EncryptionType, namespace, labelSelector string) { + t.Helper() + assertAccessTokens(t, clientSet.Etcd, string(expectedMode)) + assertAuthTokens(t, clientSet.Etcd, string(expectedMode)) + library.AssertLastMigratedKey(t, clientSet.Kube, DefaultTargetGRs, namespace, labelSelector) +} + +func AssertTokenOfLifeEncrypted(t testing.TB, clientSet library.ClientSet, rawTokenOfLife runtime.Object) { + t.Helper() + tokenOfLife := rawTokenOfLife.(*oauthapiv1.OAuthAccessToken) + rawTokenValue := GetRawTokenOfLife(t, clientSet) + if strings.Contains(rawTokenValue, tokenOfLife.RefreshToken) { + t.Errorf("access token not encrypted, token received from etcd have %q (plain text), raw content in etcd is %s", tokenOfLife.RefreshToken, rawTokenValue) + } +} + +func AssertTokenOfLifeNotEncrypted(t testing.TB, clientSet library.ClientSet, rawTokenOfLife runtime.Object) { + t.Helper() + tokenOfLife := rawTokenOfLife.(*oauthapiv1.OAuthAccessToken) + rawTokenValue := GetRawTokenOfLife(t, clientSet) + if !strings.Contains(rawTokenValue, tokenOfLife.RefreshToken) { + t.Errorf("access token received from etcd doesnt have %q (plain text), raw content in etcd is %s", tokenOfLife.RefreshToken, rawTokenValue) + } +} + +func assertAccessTokens(t testing.TB, etcdClient library.EtcdClient, expectedMode string) { + t.Logf("Checking if all OauthAccessTokens where encrypted/decrypted for %q mode", expectedMode) + totalAccessTokens, err := library.VerifyResources(t, etcdClient, "/openshift.io/oauth/accesstokens/", expectedMode, true) + t.Logf("Verified %d OauthAccessTokens", totalAccessTokens) + require.NoError(t, err) +} + +func assertAuthTokens(t testing.TB, etcdClient library.EtcdClient, expectedMode string) { + t.Logf("Checking if all OAuthAuthorizeTokens where encrypted/decrypted for %q mode", expectedMode) + totalAuthTokens, err := library.VerifyResources(t, etcdClient, "/openshift.io/oauth/authorizetokens/", expectedMode, true) + t.Logf("Verified %d OAuthAuthorizeTokens", totalAuthTokens) + require.NoError(t, err) +} diff --git a/test/e2e-encryption/oauthhelpers.go b/test/library/encryption/helpers.go similarity index 95% rename from test/e2e-encryption/oauthhelpers.go rename to test/library/encryption/helpers.go index f67801cc46..f64c9f8af9 100644 --- a/test/e2e-encryption/oauthhelpers.go +++ b/test/library/encryption/helpers.go @@ -1,4 +1,4 @@ -package e2eencryption +package encryption import ( "context" @@ -23,7 +23,7 @@ import ( ) type ClientSet struct { - OperatorClient operatorv1client.OpenShiftAPIServerInterface + OperatorClient operatorv1client.AuthenticationInterface TokenClient oauthclient.OAuthAccessTokensGetter } @@ -36,7 +36,7 @@ func GetClientsFor(t testing.TB, kubeConfig *rest.Config) ClientSet { oc, err := oauthclient.NewForConfig(kubeConfig) require.NoError(t, err) - return ClientSet{OperatorClient: operatorClient.OpenShiftAPIServers(), TokenClient: oc} + return ClientSet{OperatorClient: operatorClient.Authentications(), TokenClient: oc} } func GetClients(t testing.TB) ClientSet { diff --git a/vendor/github.com/openshift/library-go/pkg/network/dialer_linux.go b/vendor/github.com/openshift/library-go/pkg/network/dialer_linux.go index e3cd3f4d3c..b8ff8db85e 100644 --- a/vendor/github.com/openshift/library-go/pkg/network/dialer_linux.go +++ b/vendor/github.com/openshift/library-go/pkg/network/dialer_linux.go @@ -3,41 +3,38 @@ package network import ( - "context" "net" "os" "syscall" "time" "golang.org/x/sys/unix" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" ) func dialerWithDefaultOptions() DialContext { nd := &net.Dialer{ // TCP_USER_TIMEOUT does affect the behaviour of connect() which is controlled by this field so we set it to the same value Timeout: 25 * time.Second, - } - return wrapDialContext(nd.DialContext) -} - -func wrapDialContext(dc DialContext) DialContext { - return func(ctx context.Context, network, address string) (net.Conn, error) { - conn, err := dc(ctx, network, address) - if err != nil { - return conn, err - } - - if tcpCon, ok := conn.(*net.TCPConn); ok { - tcpFD, err := tcpCon.File() + // KeepAlive must to be set to a negative value to stop std library from applying the default values + // by doing so we ensure that the options we are interested in won't be overwritten + KeepAlive: time.Duration(-1), + Control: func(network, address string, con syscall.RawConn) error { + var errs []error + err := con.Control(func(fd uintptr) { + optionsErr := setDefaultSocketOptions(int(fd)) + if optionsErr != nil { + errs = append(errs, optionsErr) + } + }) if err != nil { - return conn, err + errs = append(errs, err) } - if err := setDefaultSocketOptions(int(tcpFD.Fd())); err != nil { - return conn, err - } - } - return conn, nil + return utilerrors.NewAggregate(errs) + }, } + return nd.DialContext } // setDefaultSocketOptions sets custom socket options so that we can detect connections to an unhealthy (dead) peer quickly. @@ -58,6 +55,11 @@ func setDefaultSocketOptions(fd int) error { // specifies the threshold for sending the first KEEP ALIVE probe in seconds tcpKeepIdle := int(roundDuration(2*time.Second, time.Second)) + // enable keep-alive probes + if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, 1); err != nil { + return wrapSyscallError("setsockopt", err) + } + if err := syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, tcpUserTimeoutInMilliSeconds); err != nil { return wrapSyscallError("setsockopt", err) } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/apiserver/controllerset/apiservercontrollerset.go b/vendor/github.com/openshift/library-go/pkg/operator/apiserver/controllerset/apiservercontrollerset.go index 3fdc41143e..f3c5da50aa 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/apiserver/controllerset/apiservercontrollerset.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/apiserver/controllerset/apiservercontrollerset.go @@ -66,7 +66,7 @@ type APIServerControllerSet struct { apiServiceController controllerWrapper clusterOperatorStatusController controllerWrapper configUpgradableController controllerWrapper - encryptionControllers controllerWrapper + encryptionControllers encryptionControllerBuilder finalizerController controllerWrapper logLevelController controllerWrapper pruneController controllerWrapper @@ -318,22 +318,29 @@ func (cs *APIServerControllerSet) WithEncryptionControllers( apiServerInformer configv1informers.APIServerInformer, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, ) *APIServerControllerSet { - cs.encryptionControllers.controller = encryption.NewControllers( - component, - provider, - deployer, - migrator, - cs.operatorClient, - apiServerClient, - apiServerInformer, - kubeInformersForNamespaces, - secretsClient, - cs.eventRecorder, - ) + + cs.encryptionControllers = encryptionControllerBuilder{ + operatorClient: cs.operatorClient, + eventRecorder: cs.eventRecorder, + + component: component, + provider: provider, + deployer: deployer, + migrator: migrator, + apiServerClient: apiServerClient, + apiServerInformer: apiServerInformer, + kubeInformersForNamespaces: kubeInformersForNamespaces, + secretsClient: secretsClient, + } return cs } +func (cs *APIServerControllerSet) WithUnsupportedConfigPrefixForEncryptionControllers(prefix ...string) *APIServerControllerSet { + cs.encryptionControllers.unsupportedConfigPrefix = prefix + return cs +} + func (cs *APIServerControllerSet) WithoutEncryptionControllers() *APIServerControllerSet { cs.encryptionControllers.controller = nil cs.encryptionControllers.emptyAllowed = true @@ -348,7 +355,7 @@ func (cs *APIServerControllerSet) PrepareRun() (preparedAPIServerControllerSet, "apiServiceController": cs.apiServiceController, "clusterOperatorStatusController": cs.clusterOperatorStatusController, "configUpgradableController": cs.configUpgradableController, - "encryptionControllers": cs.encryptionControllers, + "encryptionControllers": cs.encryptionControllers.build(), "finalizerController": cs.finalizerController, "logLevelController": cs.logLevelController, "pruneController": cs.pruneController, @@ -374,3 +381,42 @@ func (cs *preparedAPIServerControllerSet) Run(ctx context.Context) { go cs.controllers[i].Run(ctx, 1) } } + +type encryptionControllerBuilder struct { + controllerWrapper + + operatorClient v1helpers.OperatorClient + eventRecorder events.Recorder + + component string + provider controllers.Provider + deployer statemachine.Deployer + migrator migrators.Migrator + secretsClient corev1.SecretsGetter + apiServerClient configv1client.APIServerInterface + apiServerInformer configv1informers.APIServerInformer + kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces + + unsupportedConfigPrefix []string +} + +func (e *encryptionControllerBuilder) build() controllerWrapper { + if e.emptyAllowed { + return e.controllerWrapper + } + e.controllerWrapper.controller = encryption.NewControllers( + e.component, + e.unsupportedConfigPrefix, + e.provider, + e.deployer, + e.migrator, + e.operatorClient, + e.apiServerClient, + e.apiServerInformer, + e.kubeInformersForNamespaces, + e.secretsClient, + e.eventRecorder, + ) + + return e.controllerWrapper +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go index 77e7fad864..979edbb764 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go @@ -25,6 +25,7 @@ type runner interface { func NewControllers( component string, + unsupportedConfigPrefix []string, provider controllers.Provider, deployer statemachine.Deployer, migrator migrators.Migrator, @@ -45,6 +46,7 @@ func NewControllers( controllers: []runner{ controllers.NewKeyController( component, + unsupportedConfigPrefix, provider, deployer, operatorClient, diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go index 60b3199860..6d35545d88 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go @@ -1,6 +1,7 @@ package controllers import ( + "bytes" "context" "encoding/base64" "encoding/json" @@ -12,8 +13,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - kyaml "k8s.io/apimachinery/pkg/util/yaml" apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/klog/v2" @@ -67,10 +68,13 @@ type keyController struct { deployer statemachine.Deployer secretClient corev1client.SecretsGetter provider Provider + + unsupportedConfigPrefix []string } func NewKeyController( component string, + unsupportedConfigPrefix []string, provider Provider, deployer statemachine.Deployer, operatorClient operatorv1helpers.OperatorClient, @@ -85,8 +89,9 @@ func NewKeyController( operatorClient: operatorClient, apiServerClient: apiServerClient, - component: component, - name: "EncryptionKeyController", + component: component, + unsupportedConfigPrefix: unsupportedConfigPrefix, + name: "EncryptionKeyController", encryptionSecretSelector: encryptionSecretSelector, deployer: deployer, @@ -251,24 +256,9 @@ func (c *keyController) getCurrentModeAndExternalReason() (state.Mode, string, e return "", "", err } - // TODO make this un-settable once set - // ex: we could require the tech preview no upgrade flag to be set before we will honor this field - type unsupportedEncryptionConfig struct { - Encryption struct { - Reason string `json:"reason"` - } `json:"encryption"` - } - encryptionConfig := &unsupportedEncryptionConfig{} - if raw := operatorSpec.UnsupportedConfigOverrides.Raw; len(raw) > 0 { - jsonRaw, err := kyaml.ToJSON(raw) - if err != nil { - klog.Warning(err) - // maybe it's just json - jsonRaw = raw - } - if err := json.Unmarshal(jsonRaw, encryptionConfig); err != nil { - return "", "", err - } + encryptionConfig, err := structuredUnsupportedConfigFrom(operatorSpec.UnsupportedConfigOverrides.Raw, c.unsupportedConfigPrefix) + if err != nil { + return "", "", err } reason := encryptionConfig.Encryption.Reason @@ -336,3 +326,50 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern // this also generates back pressure for key rotation when migration takes a long time or was recently completed return latestKeyID, "rotation-interval-has-passed", time.Since(latestKey.Migrated.Timestamp) > encryptionSecretMigrationInterval } + +// TODO make this un-settable once set +// ex: we could require the tech preview no upgrade flag to be set before we will honor this field +type unsupportedEncryptionConfig struct { + Encryption struct { + Reason string `json:"reason"` + } `json:"encryption"` +} + +// structuredUnsupportedConfigFrom returns unsupportedEncryptionConfig from the operator's observedConfig +func structuredUnsupportedConfigFrom(rawConfig []byte, prefix []string) (unsupportedEncryptionConfig, error) { + if len(rawConfig) == 0 { + return unsupportedEncryptionConfig{}, nil + } + + unstructuredRawJSONCfg, err := unstructuredUnsupportedConfigFromWithPrefix(rawConfig, prefix) + if err != nil { + return unsupportedEncryptionConfig{}, err + } + + encryptionConfig := unsupportedEncryptionConfig{} + if err := json.Unmarshal(unstructuredRawJSONCfg, &encryptionConfig); err != nil { + return unsupportedEncryptionConfig{}, err + } + + return encryptionConfig, nil +} + +// unstructuredUnsupportedConfigFrom returns the configuration from the operator's observedConfig field in the subtree given by the prefix +func unstructuredUnsupportedConfigFromWithPrefix(rawConfig []byte, prefix []string) ([]byte, error) { + if len(prefix) == 0 { + return rawConfig, nil + } + + prefixedConfig := map[string]interface{}{} + if err := json.NewDecoder(bytes.NewBuffer(rawConfig)).Decode(&prefixedConfig); err != nil { + klog.V(4).Infof("decode of existing config failed with error: %v", err) + return nil, err + } + + actualConfig, _, err := unstructured.NestedFieldCopy(prefixedConfig, prefix...) + if err != nil { + return nil, err + } + + return json.Marshal(actualConfig) +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go index 6c4c19ae46..15058c29da 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go @@ -91,6 +91,24 @@ func getDesiredEncryptionState(oldEncryptionConfig *apiserverconfigv1.Encryption } } + // remove unused GRs from the desired encryption configuration + // toBeEncryptedGRs is not static and can change over time + // here we are removing resources that his operator doesn't manage anymore + for actualGR := range desiredEncryptionState { + found := false + for _, desiredGR := range toBeEncryptedGRs { + if actualGR == desiredGR { + found = true + break + } + } + + if !found { + delete(desiredEncryptionState, actualGR) + klog.V(4).Infof("removed %s from the encryption config as this operator doesn't manage this GR anymore", actualGR.String()) + } + } + // // STEP 1: without secrets, wait for the key controller to create one // @@ -198,5 +216,6 @@ func getDesiredEncryptionState(oldEncryptionConfig *apiserverconfigv1.Encryption desiredEncryptionState[gr] = grState } klog.V(4).Infof("write key %s set as sole write key", writeKey.Key.Name) + return desiredEncryptionState } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/resource/resourceapply/monitoring.go b/vendor/github.com/openshift/library-go/pkg/operator/resource/resourceapply/monitoring.go index 6525ca5eb0..5b297eb7f4 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/resource/resourceapply/monitoring.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/resource/resourceapply/monitoring.go @@ -76,6 +76,9 @@ func ApplyServiceMonitor(client dynamic.Interface, recorder events.Recorder, ser recorder.Eventf("ServiceMonitorCreated", "Created ServiceMonitor.monitoring.coreos.com/v1 because it was missing") return true, nil } + if err != nil { + return false, err + } existingCopy := existing.DeepCopy() diff --git a/vendor/modules.txt b/vendor/modules.txt index 966d994175..702fe31c1b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -193,7 +193,7 @@ github.com/openshift/client-go/route/informers/externalversions/internalinterfac github.com/openshift/client-go/route/informers/externalversions/route github.com/openshift/client-go/route/informers/externalversions/route/v1 github.com/openshift/client-go/route/listers/route/v1 -# github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08 +# github.com/openshift/library-go v0.0.0-20201202092334-3557e59845d7 ## explicit github.com/openshift/library-go/pkg/apps/deployment github.com/openshift/library-go/pkg/assets