From 62dde189a19cc894c71ec8d0d6a794881222cda2 Mon Sep 17 00:00:00 2001 From: liuxu Date: Tue, 10 Nov 2026 19:14:12 +0800 Subject: [PATCH] feat(usagedb): add authentication to UsageDBConfig The usage database client has no way to authenticate: it is built from an address alone, a credential in connectionString is plain text in the SchedulingShard and in the scheduler ConfigMap, and connectionStringEnvVar names a variable the operator gives no way to set. Add UsageDBConfig.Auth with two mutually exclusive forms, in the shape the Prometheus Operator uses: - basicAuth: a username and an optional password - authorization: a type (default Bearer) and credentials Each value has one source, a Secret key or a file. On a SchedulingShard the source is a Secret key; the operator can mount it and give the scheduler the file path, so no credential reaches the scheduler configuration. A scheduler that runs without the operator sets the file paths directly. Auth.Validate checks that the two forms are not combined, that each value has exactly one source, that a Secret reference has a name and a key, and that "Basic" is not used as an authorization type. Refs kai-scheduler/KAI-Scheduler#2639 Co-Authored-By: Claude Fable 5.2 Signed-off-by: liuxu --- .../unreleased/added-20261110-191258.yaml | 3 + .../crd/kai.scheduler_schedulingshards.yaml | 112 ++++++++++ usagedb/auth.go | 177 +++++++++++++++ usagedb/auth_test.go | 208 ++++++++++++++++++ usagedb/usagedb.go | 12 +- 5 files changed, 508 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/added-20261110-191258.yaml create mode 100644 usagedb/auth.go create mode 100644 usagedb/auth_test.go diff --git a/.changes/unreleased/added-20261110-191258.yaml b/.changes/unreleased/added-20261110-191258.yaml new file mode 100644 index 0000000..ad1b376 --- /dev/null +++ b/.changes/unreleased/added-20261110-191258.yaml @@ -0,0 +1,3 @@ +kind: Added +body: |- + `usagedb.UsageDBConfig.Auth`: basic auth or an `Authorization` header for the usage database client, with the credentials taken from Secret keys or files, and `Auth.Validate` to check them diff --git a/config/crd/kai.scheduler_schedulingshards.yaml b/config/crd/kai.scheduler_schedulingshards.yaml index 84d2020..2547ab4 100644 --- a/config/crd/kai.scheduler_schedulingshards.yaml +++ b/config/crd/kai.scheduler_schedulingshards.yaml @@ -199,6 +199,118 @@ spec: description: UsageDBConfig defines configuration for the usage db client properties: + auth: + description: |- + Auth configures how the client authenticates to the usage database. + If not set, requests carry no credentials. + properties: + authorization: + description: |- + Authorization authenticates with an "Authorization: " + header, such as a bearer token. + properties: + credentials: + description: Credentials is the Secret key that holds + the credentials. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + credentialsFile: + description: |- + CredentialsFile is the path of a file that holds the credentials. The + operator sets it in the scheduler configuration from Credentials. + type: string + type: + description: |- + Type is the authentication type. Defaults to "Bearer". "Basic" is not + accepted: use BasicAuth. + type: string + type: object + basicAuth: + description: BasicAuth authenticates with a username and a + password. + properties: + password: + description: Password is the Secret key that holds the + password. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + passwordFile: + description: |- + PasswordFile is the path of a file that holds the password. The operator + sets it in the scheduler configuration from Password. + type: string + username: + description: Username is the Secret key that holds the + username. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + usernameFile: + description: |- + UsernameFile is the path of a file that holds the username. The operator + sets it in the scheduler configuration from Username. + type: string + type: object + type: object clientType: type: string connectionString: diff --git a/usagedb/auth.go b/usagedb/auth.go new file mode 100644 index 0000000..fd2eb8e --- /dev/null +++ b/usagedb/auth.go @@ -0,0 +1,177 @@ +// Copyright 2025 NVIDIA CORPORATION +// SPDX-License-Identifier: Apache-2.0 + +package usagedb + +import ( + "errors" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" +) + +// DefaultAuthorizationType is the Authorization type used when none is set. +const DefaultAuthorizationType = "Bearer" + +// Auth configures how the usage db client authenticates to the usage database. +// BasicAuth and Authorization are mutually exclusive. +// +// A credential is never written inline. On a SchedulingShard it is a key of a +// Secret in the namespace KAI runs in: the operator mounts that key into the +// scheduler pod and gives the scheduler the path of the mounted file. A +// scheduler that runs without the operator sets the file paths itself. +type Auth struct { + // BasicAuth authenticates with a username and a password. + // +kubebuilder:validation:Optional + BasicAuth *BasicAuth `yaml:"basicAuth,omitempty" json:"basicAuth,omitempty"` + // Authorization authenticates with an "Authorization: " + // header, such as a bearer token. + // +kubebuilder:validation:Optional + Authorization *Authorization `yaml:"authorization,omitempty" json:"authorization,omitempty"` +} + +// BasicAuth holds the sources of a basic auth username and password. The +// username is required and has exactly one source; the password is optional. +type BasicAuth struct { + // Username is the Secret key that holds the username. + // +kubebuilder:validation:Optional + Username *corev1.SecretKeySelector `yaml:"username,omitempty" json:"username,omitempty"` + // Password is the Secret key that holds the password. + // +kubebuilder:validation:Optional + Password *corev1.SecretKeySelector `yaml:"password,omitempty" json:"password,omitempty"` + // UsernameFile is the path of a file that holds the username. The operator + // sets it in the scheduler configuration from Username. + // +kubebuilder:validation:Optional + UsernameFile string `yaml:"usernameFile,omitempty" json:"usernameFile,omitempty"` + // PasswordFile is the path of a file that holds the password. The operator + // sets it in the scheduler configuration from Password. + // +kubebuilder:validation:Optional + PasswordFile string `yaml:"passwordFile,omitempty" json:"passwordFile,omitempty"` +} + +// Authorization holds the type and the source of the credentials of an +// Authorization header. The credentials have exactly one source. +type Authorization struct { + // Type is the authentication type. Defaults to "Bearer". "Basic" is not + // accepted: use BasicAuth. + // +kubebuilder:validation:Optional + Type string `yaml:"type,omitempty" json:"type,omitempty"` + // Credentials is the Secret key that holds the credentials. + // +kubebuilder:validation:Optional + Credentials *corev1.SecretKeySelector `yaml:"credentials,omitempty" json:"credentials,omitempty"` + // CredentialsFile is the path of a file that holds the credentials. The + // operator sets it in the scheduler configuration from Credentials. + // +kubebuilder:validation:Optional + CredentialsFile string `yaml:"credentialsFile,omitempty" json:"credentialsFile,omitempty"` +} + +// GetType returns the authentication type, or DefaultAuthorizationType if none is set. +func (a *Authorization) GetType() string { + if a == nil || strings.TrimSpace(a.Type) == "" { + return DefaultAuthorizationType + } + return strings.TrimSpace(a.Type) +} + +// Validate reports whether the configuration names a usable set of credential +// sources. A nil or empty Auth is valid and means no authentication. +func (a *Auth) Validate() error { + if a == nil { + return nil + } + if a.BasicAuth != nil && a.Authorization != nil { + return errors.New("basicAuth and authorization are mutually exclusive") + } + if a.BasicAuth != nil { + if err := a.BasicAuth.validate(); err != nil { + return fmt.Errorf("basicAuth: %w", err) + } + } + if a.Authorization != nil { + if err := a.Authorization.validate(); err != nil { + return fmt.Errorf("authorization: %w", err) + } + } + return nil +} + +func (b *BasicAuth) validate() error { + hasUsername, err := hasOneSource("username", b.Username, "usernameFile", b.UsernameFile) + if err != nil { + return err + } + if !hasUsername { + return errors.New("one of username and usernameFile is required") + } + _, err = hasOneSource("password", b.Password, "passwordFile", b.PasswordFile) + return err +} + +func (a *Authorization) validate() error { + if strings.EqualFold(a.GetType(), "Basic") { + return errors.New(`type "Basic" is not accepted, use basicAuth`) + } + hasCredentials, err := hasOneSource("credentials", a.Credentials, "credentialsFile", a.CredentialsFile) + if err != nil { + return err + } + if !hasCredentials { + return errors.New("one of credentials and credentialsFile is required") + } + return nil +} + +// hasOneSource reports whether a value has a source, and fails if it has both +// or if its Secret reference is incomplete. +func hasOneSource( + selectorField string, selector *corev1.SecretKeySelector, fileField string, file string, +) (bool, error) { + if selector != nil && file != "" { + return false, fmt.Errorf("%s and %s are mutually exclusive", selectorField, fileField) + } + if selector != nil && (selector.Name == "" || selector.Key == "") { + return false, fmt.Errorf("%s needs both a Secret name and a key", selectorField) + } + return selector != nil || file != "", nil +} + +// DeepCopy returns a copy that shares no memory with the receiver. +func (a *Auth) DeepCopy() *Auth { + if a == nil { + return nil + } + out := new(Auth) + out.BasicAuth = a.BasicAuth.DeepCopy() + out.Authorization = a.Authorization.DeepCopy() + return out +} + +// DeepCopy returns a copy that shares no memory with the receiver. +func (b *BasicAuth) DeepCopy() *BasicAuth { + if b == nil { + return nil + } + out := new(BasicAuth) + *out = *b + if b.Username != nil { + out.Username = b.Username.DeepCopy() + } + if b.Password != nil { + out.Password = b.Password.DeepCopy() + } + return out +} + +// DeepCopy returns a copy that shares no memory with the receiver. +func (a *Authorization) DeepCopy() *Authorization { + if a == nil { + return nil + } + out := new(Authorization) + *out = *a + if a.Credentials != nil { + out.Credentials = a.Credentials.DeepCopy() + } + return out +} diff --git a/usagedb/auth_test.go b/usagedb/auth_test.go new file mode 100644 index 0000000..8fe52d8 --- /dev/null +++ b/usagedb/auth_test.go @@ -0,0 +1,208 @@ +// Copyright 2025 NVIDIA CORPORATION +// SPDX-License-Identifier: Apache-2.0 + +package usagedb + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" +) + +func secretKey(name, key string) *corev1.SecretKeySelector { + return &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + Key: key, + } +} + +func TestAuth_Validate(t *testing.T) { + tests := []struct { + name string + auth *Auth + wantErr string + }{ + {name: "nil auth means no authentication", auth: nil}, + {name: "empty auth means no authentication", auth: &Auth{}}, + { + name: "basic auth from secret keys", + auth: &Auth{BasicAuth: &BasicAuth{ + Username: secretKey("usage-db", "username"), + Password: secretKey("usage-db", "password"), + }}, + }, + { + name: "basic auth from files", + auth: &Auth{BasicAuth: &BasicAuth{UsernameFile: "/auth/username", PasswordFile: "/auth/password"}}, + }, + { + name: "basic auth without a password", + auth: &Auth{BasicAuth: &BasicAuth{Username: secretKey("usage-db", "username")}}, + }, + { + name: "bearer token from a secret key", + auth: &Auth{Authorization: &Authorization{Credentials: secretKey("usage-db", "token")}}, + }, + { + name: "custom authorization type from a file", + auth: &Auth{Authorization: &Authorization{Type: "Token", CredentialsFile: "/auth/credentials"}}, + }, + { + name: "basic auth and authorization together", + auth: &Auth{ + BasicAuth: &BasicAuth{Username: secretKey("usage-db", "username")}, + Authorization: &Authorization{Credentials: secretKey("usage-db", "token")}, + }, + wantErr: "basicAuth and authorization are mutually exclusive", + }, + { + name: "basic auth without a username", + auth: &Auth{BasicAuth: &BasicAuth{Password: secretKey("usage-db", "password")}}, + wantErr: "basicAuth: one of username and usernameFile is required", + }, + { + name: "username from both a secret key and a file", + auth: &Auth{BasicAuth: &BasicAuth{ + Username: secretKey("usage-db", "username"), + UsernameFile: "/auth/username", + }}, + wantErr: "basicAuth: username and usernameFile are mutually exclusive", + }, + { + name: "password from both a secret key and a file", + auth: &Auth{BasicAuth: &BasicAuth{ + Username: secretKey("usage-db", "username"), + Password: secretKey("usage-db", "password"), + PasswordFile: "/auth/password", + }}, + wantErr: "basicAuth: password and passwordFile are mutually exclusive", + }, + { + name: "secret reference without a key", + auth: &Auth{BasicAuth: &BasicAuth{Username: secretKey("usage-db", "")}}, + wantErr: "basicAuth: username needs both a Secret name and a key", + }, + { + name: "secret reference without a name", + auth: &Auth{Authorization: &Authorization{Credentials: secretKey("", "token")}}, + wantErr: "authorization: credentials needs both a Secret name and a key", + }, + { + name: "authorization without credentials", + auth: &Auth{Authorization: &Authorization{Type: "Bearer"}}, + wantErr: "authorization: one of credentials and credentialsFile is required", + }, + { + name: "credentials from both a secret key and a file", + auth: &Auth{Authorization: &Authorization{ + Credentials: secretKey("usage-db", "token"), + CredentialsFile: "/auth/credentials", + }}, + wantErr: "authorization: credentials and credentialsFile are mutually exclusive", + }, + { + name: "basic as an authorization type", + auth: &Auth{Authorization: &Authorization{Type: "basic", Credentials: secretKey("usage-db", "token")}}, + wantErr: `authorization: type "Basic" is not accepted, use basicAuth`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.auth.Validate() + if tt.wantErr == "" { + assert.NoError(t, err) + return + } + assert.EqualError(t, err, tt.wantErr) + }) + } +} + +func TestAuthorization_GetType(t *testing.T) { + var unset *Authorization + assert.Equal(t, DefaultAuthorizationType, unset.GetType()) + assert.Equal(t, DefaultAuthorizationType, (&Authorization{}).GetType()) + assert.Equal(t, DefaultAuthorizationType, (&Authorization{Type: " "}).GetType()) + assert.Equal(t, "Token", (&Authorization{Type: " Token "}).GetType()) +} + +func TestUsageDBConfig_DeepCopy_Auth(t *testing.T) { + t.Run("no auth stays nil", func(t *testing.T) { + copied := (&UsageDBConfig{ClientType: "prometheus"}).DeepCopy() + assert.Nil(t, copied.Auth) + }) + + t.Run("basic auth is copied, not shared", func(t *testing.T) { + original := &UsageDBConfig{ + ClientType: "prometheus", + Auth: &Auth{BasicAuth: &BasicAuth{ + Username: secretKey("usage-db", "username"), + Password: secretKey("usage-db", "password"), + UsernameFile: "/auth/username", + PasswordFile: "/auth/password", + }}, + } + copied := original.DeepCopy() + require.Equal(t, original, copied) + + copied.Auth.BasicAuth.Username.Name = "other" + copied.Auth.BasicAuth.Password.Key = "other" + copied.Auth.BasicAuth.PasswordFile = "/other" + assert.Equal(t, "usage-db", original.Auth.BasicAuth.Username.Name) + assert.Equal(t, "password", original.Auth.BasicAuth.Password.Key) + assert.Equal(t, "/auth/password", original.Auth.BasicAuth.PasswordFile) + }) + + t.Run("authorization is copied, not shared", func(t *testing.T) { + original := &UsageDBConfig{ + ClientType: "prometheus", + Auth: &Auth{Authorization: &Authorization{ + Type: "Bearer", + Credentials: secretKey("usage-db", "token"), + }}, + } + copied := original.DeepCopy() + require.Equal(t, original, copied) + + copied.Auth.Authorization.Credentials.Key = "other" + copied.Auth.Authorization.Type = "Token" + assert.Equal(t, "token", original.Auth.Authorization.Credentials.Key) + assert.Equal(t, "Bearer", original.Auth.Authorization.Type) + }) +} + +// The scheduler configuration and the SchedulingShard are both read as JSON +// (sigs.k8s.io/yaml), so the JSON field names are the contract. +func TestUsageDBConfig_Auth_JSON(t *testing.T) { + const document = `{ + "clientType": "prometheus", + "connectionString": "https://metrics.example.com/prometheus", + "auth": { + "basicAuth": { + "username": {"name": "usage-db", "key": "username"}, + "password": {"name": "usage-db", "key": "password"} + } + } + }` + + var config UsageDBConfig + require.NoError(t, json.Unmarshal([]byte(document), &config)) + require.NotNil(t, config.Auth) + require.NotNil(t, config.Auth.BasicAuth) + assert.Equal(t, secretKey("usage-db", "username"), config.Auth.BasicAuth.Username) + assert.Equal(t, secretKey("usage-db", "password"), config.Auth.BasicAuth.Password) + assert.NoError(t, config.Auth.Validate()) + + encoded, err := json.Marshal(&UsageDBConfig{ + ClientType: "prometheus", + Auth: &Auth{Authorization: &Authorization{CredentialsFile: "/auth/credentials"}}, + }) + require.NoError(t, err) + assert.JSONEq(t, + `{"clientType":"prometheus","auth":{"authorization":{"credentialsFile":"/auth/credentials"}}}`, + string(encoded)) +} diff --git a/usagedb/usagedb.go b/usagedb/usagedb.go index f35760a..d7807b8 100644 --- a/usagedb/usagedb.go +++ b/usagedb/usagedb.go @@ -9,10 +9,13 @@ import ( ) type UsageDBConfig struct { - ClientType string `yaml:"clientType" json:"clientType"` - ConnectionString string `yaml:"connectionString,omitempty" json:"connectionString,omitempty"` - ConnectionStringEnvVar string `yaml:"connectionStringEnvVar,omitempty" json:"connectionStringEnvVar,omitempty"` - UsageParams *UsageParams `yaml:"usageParams,omitempty" json:"usageParams,omitempty"` + ClientType string `yaml:"clientType" json:"clientType"` + ConnectionString string `yaml:"connectionString,omitempty" json:"connectionString,omitempty"` + ConnectionStringEnvVar string `yaml:"connectionStringEnvVar,omitempty" json:"connectionStringEnvVar,omitempty"` + // Auth configures how the client authenticates to the usage database. + // If not set, requests carry no credentials. + Auth *Auth `yaml:"auth,omitempty" json:"auth,omitempty"` + UsageParams *UsageParams `yaml:"usageParams,omitempty" json:"usageParams,omitempty"` } // GetUsageParams returns the usage params if set, and default params if not set. @@ -30,6 +33,7 @@ func (c *UsageDBConfig) DeepCopy() *UsageDBConfig { out.ClientType = c.ClientType out.ConnectionString = c.ConnectionString out.ConnectionStringEnvVar = c.ConnectionStringEnvVar + out.Auth = c.Auth.DeepCopy() if c.UsageParams != nil { out.UsageParams = c.UsageParams.DeepCopy() }