Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,13 @@ test-e2e-encryption: GO_TEST_FLAGS += -parallel 1
test-e2e-encryption: test-unit
.PHONY: test-e2e-encryption

test-e2e-encryption-perf: GO_TEST_PACKAGES :=./test/e2e-encryption-perf/...
test-e2e-encryption-perf: GO_TEST_FLAGS += -v
test-e2e-encryption-perf: GO_TEST_FLAGS += -timeout 1h
test-e2e-encryption-perf: GO_TEST_FLAGS += -p 1
test-e2e-encryption-perf: test-unit
.PHONY: test-e2e-encryption-perf

# Configure the 'telepresence' target
# See vendor/github.com/openshift/build-machinery-go/scripts/run-telepresence.sh for usage and configuration details
export TP_DEPLOYMENT_YAML ?=./manifests/07_deployment.yaml
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ require (
github.com/openshift/api v0.0.0-20201019163320-c6a5ec25f267
github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab
github.com/openshift/client-go v0.0.0-20201020074620-f8fd44879f7c
github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08
github.com/openshift/library-go v0.0.0-20201202092334-3557e59845d7
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -353,8 +353,8 @@ github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab h1:lB
github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
github.com/openshift/client-go v0.0.0-20201020074620-f8fd44879f7c h1:NB9g4Y/aegId7fyNqYyGxEfyNOytYFT5dxWJtfOJFQs=
github.com/openshift/client-go v0.0.0-20201020074620-f8fd44879f7c/go.mod h1:yZ3u8vgWC19I9gbDMRk8//9JwG/0Sth6v7C+m6R8HXs=
github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08 h1:Z+8t3ooTH2T+J/GoCZbgaOk5WqNZgPuHlUAKMfG1FEk=
github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08/go.mod h1:1xYaYQcQsn+AyCRsvOU+Qn5z6GGiCmcblXkT/RZLVfo=
github.com/openshift/library-go v0.0.0-20201202092334-3557e59845d7 h1:haE99sHkbQs+QPIFHKgV8RKZ9EuBRFXraIwwEi0rX98=
github.com/openshift/library-go v0.0.0-20201202092334-3557e59845d7/go.mod h1:1xYaYQcQsn+AyCRsvOU+Qn5z6GGiCmcblXkT/RZLVfo=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
Expand Down
6 changes: 5 additions & 1 deletion pkg/operator/encryptionprovider/encryptionprovider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,14 @@ func TestEncryptionProvider(t *testing.T) {
expectedShouldRunEncryptionCtrl bool
}{
{
// this will be default in 4.7
name: "encryption off, default GRs returned",
defaultEncryptedGRs: defaultGRs,
expectedShouldRunEncryptionCtrl: false,
expectedEncryptedGRs: defaultGRs,
expectedShouldRunEncryptionCtrl: true,
},
{
// it means that encryption was on in 4.7 or the cluster was updated from 4.6 and the annotation was removed
name: "encryption on, secret without the annotation, default GRs returned",
initialSecrets: []*corev1.Secret{
func() *corev1.Secret {
Expand All @@ -48,6 +51,7 @@ func TestEncryptionProvider(t *testing.T) {
expectedShouldRunEncryptionCtrl: true,
},
{
// it means that encryption was on in 4.6 and migration hasn't finished yet
name: "encryption on, secret with the annotation, reduced GRs returned",
initialSecrets: []*corev1.Secret{defaultSecret("openshift-apiserver", encryptionCfgAnnotationKey)},
defaultEncryptedGRs: defaultGRs,
Expand Down
30 changes: 18 additions & 12 deletions pkg/operator/encryptionprovider/encyrptionprovider.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,16 @@ func New(
// EncryptedGRs returns resources that need to be encrypted
// Note: the list can change depending on the existence and attached annotations of encryption-config-openshift-oauth-apiserver in openshift-config-managed namespace as described in https://github.com/openshift/enhancements/blob/master/enhancements/etcd/etcd-encryption-for-separate-oauth-apis.md
//
// case 1 encryption off or the secret was annotated - return an empty list of EncryptedGRs
// case 2 otherwise return the authoritative list of EncryptedGRs
// case 1 encryption off OR the encryption-config-openshift-oauth-apiserver doesn't have "encryption.apiserver.operator.openshift.io/managed-by" annotation
// - we will the authoritative list of EncryptedGRs
// - at the time of this writhing that list contains:
// {Group: "oauth.openshift.io", Resource: "oauthaccesstokens"}
// {Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"}
//
// TODO:
// - change the code in 4.7 to return a static list (the authoritative list)
// case 2 otherwise (the encryption config doesn't have the annotation) return an empty list of EncryptedGRs - that means that OAS-O is still in charge of the encryption config
//
// TODO in 4.8
// this encryption provided won't be needed and can be removed
func (p *encryptionProvider) EncryptedGRs() []schema.GroupResource {
inCharge, err := p.isOAuthEncryptionConfigManagedByThisOperator()
if err != nil || !inCharge {
Expand All @@ -59,12 +64,13 @@ func (p *encryptionProvider) ShouldRunEncryptionControllers() (bool, error) {
return p.isOAuthEncryptionConfigManagedByThisOperator()
}

// isOAuthEncryptionConfigManagedByThisOperator determines whether this operator is in charge of encryption-config-openshift-oauth-apiserver
// isOAuthEncryptionConfigManagedByThisOperator determines whether this operator is in charge of encryption-config-openshift-oauth-apiserver in openshift-config-managed namespace
//
// case 1 encryption off OR the encryption config doesn't have "encryption.apiserver.operator.openshift.io/managed-by" annotation - this operator is in charge
// case 2 the encryption config have the annotation - OAS-O is still in charge
//
// case 1 encryption off or the secret was annotated - OAS-O is in charge
// case 2 otherwise this operator will manage its own encryption configuration
// TODO:
// - change the case 1 in 4.7 so that CAO manages its own encryption config when encryption is off
// TODO in 4.8
// this encryption provider won't be needed and can be removed
func (p *encryptionProvider) isOAuthEncryptionConfigManagedByThisOperator() (bool, error) {
oauthAPIServerEncryptionCfgName := fmt.Sprintf("%s-%s", encryptionconfig.EncryptionConfSecretName, p.targetNamespace)
oauthAPIServerEncryptionCfg, err := p.secretLister.Get(oauthAPIServerEncryptionCfgName)
Expand All @@ -74,10 +80,10 @@ func (p *encryptionProvider) isOAuthEncryptionConfigManagedByThisOperator() (boo
// - we suppress the error so that the encryption controllers:
// 1. don't report Degraded when encryption is off
// 2. don't requeue when encryption is off
return false, nil // case 1 - OAS-O in charge

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see first line of the comment does not match. Probably the whole block of comment needs an update.

return true, nil // case 1 - we are in charge
}
if _, exist := oauthAPIServerEncryptionCfg.Annotations[p.encryptionCfgAnnotationKey]; exist {
return false, nil // case 1 - OAS-O in charge
return false, nil // case 2 - OAS-O is in charge
}
return true, nil // case 2 - taking over
return true, nil // case 1 - taking over
}
2 changes: 2 additions & 0 deletions pkg/operator/starter.go
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,8 @@ func prepareOauthAPIServerOperator(ctx context.Context, controllerContext *contr
operatorCtx.configClient.ConfigV1().APIServers(),
operatorCtx.operatorConfigInformer.Config().V1().APIServers(),
operatorCtx.kubeInformersForNamespaces,
).WithUnsupportedConfigPrefixForEncryptionControllers(
oauthapiconfigobservercontroller.OAuthAPIServerConfigPrefix,
).WithFinalizerController(
"openshift-oauth-apiserver",
operatorCtx.kubeInformersForNamespaces.InformersFor("openshift-oauth-apiserver"),
Expand Down
105 changes: 105 additions & 0 deletions test/e2e-encryption-perf/encryption_perf_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package e2e_encryption_perf

import (
"context"
"errors"
"fmt"
"testing"
"time"

"github.com/stretchr/testify/require"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/client-go/kubernetes"

oauthapiv1 "github.com/openshift/api/oauth/v1"
operatorv1 "github.com/openshift/api/operator/v1"
oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1"
operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library"
operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption"
library "github.com/openshift/library-go/test/library/encryption"
)

const (
tokenStatsKey = "created oauthaccesstokens"
)

func TestPerfEncryptionTypeAESCBC(tt *testing.T) {
ctx := context.TODO()
clientSet := getPerfClients(tt)
library.TestPerfEncryptionTypeAESCBC(tt, library.PerfScenario{
BasicScenario: library.BasicScenario{
Namespace: "openshift-config-managed",
LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver",
EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"),
EncryptionConfigSecretNamespace: "openshift-config-managed",
OperatorNamespace: "openshift-authentication-operator",
TargetGRs: operatorencryption.DefaultTargetGRs,
AssertFunc: operatorencryption.AssertTokens,
},
GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) {
apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{})
if err != nil {
return nil, err
}
return apiServerOperator.Status.Conditions, nil
},
AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) {
tokenCount, ok := statStore[tokenStatsKey]
if !ok {
err := errors.New("missing oauth access tokens count stats, can't continue the test")
require.NoError(t, err)
}
if tokenCount < 14000 {
err := fmt.Errorf("expected to create at least 14000 tokens but %d were created", tokenCount)
require.NoError(t, err)
}
t.Logf("Created %d access tokens", tokenCount)
},
AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) {
t.Logf("migration took %v", migrationTime)
expectedMigrationTime := 10 * time.Minute
if migrationTime > expectedMigrationTime {
t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime)
}
},
DBLoaderWorkers: 3,
DBLoaderFunc: library.DBLoaderRepeat(1, false,
library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)),
})
}

func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType {
return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error {
token := &oauthapiv1.OAuthAccessToken{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("encryption-token-%v", rand.String(18)),
},
RefreshToken: "I have no special talents. I am only passionately curious",
UserName: "kube:admin",
Scopes: []string{"user:full"},
RedirectURI: "redirect.me.to.token.of.life",
ClientName: "console",
UserUID: "non-existing-user-id",
}
_, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{})
return err
}
}

func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error {
statsCollector(tokenStatsKey)
return nil
}

func getPerfClients(t *testing.T) operatorencryption.ClientSet {
t.Helper()

kubeConfig := operatorlibrary.NewClientConfigForTest(t)

kubeConfig.QPS = 300
kubeConfig.Burst = 600

return operatorencryption.GetClientsFor(t, kubeConfig)
}
Loading