From c425029bffac6d37edc1d6765c8c1948a6a84780 Mon Sep 17 00:00:00 2001 From: Rahul Singhal Date: Thu, 20 Aug 2026 00:31:21 +0000 Subject: [PATCH 1/4] feat(kernel): support JWT private-key M2M auth via WithJWTPrivateKeyM2M MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add OAuth M2M auth with a JWT private-key client assertion (RFC 7523) on the kernel backend. The kernel signs a short-lived assertion with the private key instead of sending a client secret and owns the token lifecycle. Kernel-backend only (the pure-Go Thrift path can't sign the assertion). - auth/oauth/jwtm2m: new authenticator satisfying the (internal) JWTM2MCredentialsProvider interface; Authenticate returns a clear kernel-only error on the Thrift path. - internal/backend/kernel: AuthJWTM2M mode + Auth JWT fields + JWTM2MCredentialsProvider interface; setAuth maps it to the kernel_session_config_set_auth_m2m_jwt C setter (8 args incl. token_url). - kernel_config: resolveKernelAuth resolves the JWT authenticator to the descriptor (scopes + token_url forwarded, unlike the scopes-less shared-secret M2M setter). - connector: WithJWTPrivateKeyM2M(JWTPrivateKeyM2MConfig{...}) — a struct option (not positional args) so the many fields can't be transposed. - Bump KERNEL_REV to a build with the JWT C-ABI setter; update the TestSetRetry max --- CHANGELOG.md | 1 + KERNEL_REV | 2 +- auth/oauth/jwtm2m/jwtm2m.go | 63 ++++++++++++++++++++++ auth/oauth/jwtm2m/jwtm2m_test.go | 45 ++++++++++++++++ connector.go | 47 +++++++++++++++++ doc.go | 7 ++- internal/backend/kernel/auth.go | 34 ++++++++++-- internal/backend/kernel/backend.go | 27 ++++++++++ internal/backend/kernel/kernel_test.go | 32 +++++++++--- kernel_config.go | 16 ++++++ kernel_config_test.go | 62 ++++++++++++++++++++++ kernel_jwt_e2e_test.go | 72 ++++++++++++++++++++++++++ 12 files changed, 394 insertions(+), 14 deletions(-) create mode 100644 auth/oauth/jwtm2m/jwtm2m.go create mode 100644 auth/oauth/jwtm2m/jwtm2m_test.go create mode 100644 kernel_jwt_e2e_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8acc5d8f..29d5affb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History ## Unreleased +- Kernel backend (`WithUseKernel(true)`): OAuth **M2M with a JWT private-key client assertion** (RFC 7523) is now supported via `WithJWTPrivateKeyM2M(JWTPrivateKeyM2MConfig{...})`. The kernel signs a short-lived assertion with the private key instead of sending a client secret and owns the token lifecycle. `ClientID`, `KeyFile`, and `Kid` are required; `Passphrase` (encrypted PKCS#8 key), `Algorithm` (default `RS256`), `TokenURL` (the OAuth IdP token endpoint — required for an external-IdP-backed workspace such as Entra ID, since Databricks-native OIDC does not advertise the `private_key_jwt` method), and `Scopes` are optional. Kernel-backend only; on the default (Thrift) backend a connection built with this authenticator fails at authenticate time with a clear kernel-only error. Verified end-to-end against an Azure Databricks warehouse via Entra ID. Also bumps the pinned `KERNEL_REV` to a build that includes the JWT C-ABI setter. DSN-string configuration (`authType=...`) for JWT M2M is deferred to a follow-up; use the programmatic option. - Improve telemetry error reporting: driver failures are now categorized by cause instead of reported as a generic error (databricks/databricks-sql-go#414, #415, #417, #419, #424) ## v1.14.0 (2026-07-13) diff --git a/KERNEL_REV b/KERNEL_REV index a8ceb742..8ea7940e 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -9ac3f3d3f3e804d52d8890e890d9f8a8a617ec93 +ef2c8bedb0345076d3147e05c35362778ac4e130 diff --git a/auth/oauth/jwtm2m/jwtm2m.go b/auth/oauth/jwtm2m/jwtm2m.go new file mode 100644 index 00000000..bc71f2f0 --- /dev/null +++ b/auth/oauth/jwtm2m/jwtm2m.go @@ -0,0 +1,63 @@ +// Package jwtm2m provides an OAuth machine-to-machine authenticator that +// authenticates with a JWT private-key client assertion (RFC 7523) instead of +// a client secret. +// +// This authenticator is KERNEL-BACKEND ONLY. The JWT is signed by the native +// kernel (which owns the assertion signing + token lifecycle); the pure-Go +// Thrift path has no JWT-signing implementation, so Authenticate returns an +// error directing the caller at the kernel backend. The authenticator exists +// so cfg.Authenticator stays the single source of truth for auth on both +// backends: the kernel selects this mode by asserting the (internal) +// JWTM2MCredentialsProvider interface that this type satisfies structurally. +package jwtm2m + +import ( + "fmt" + "net/http" + + "github.com/databricks/databricks-sql-go/auth" +) + +// NewAuthenticator builds a JWT private-key M2M authenticator. clientID, +// keyFile, and kid are required; passphrase (for an encrypted PKCS#8 key), +// algorithm (default RS256), tokenURL (the OAuth IdP token endpoint — required +// for an external-IdP-backed workspace such as Entra ID), and scopes are +// optional. +func NewAuthenticator(clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string) auth.Authenticator { + return &authClient{ + clientID: clientID, + keyFile: keyFile, + kid: kid, + passphrase: passphrase, + algorithm: algorithm, + tokenURL: tokenURL, + scopes: scopes, + } +} + +type authClient struct { + clientID string + keyFile string + kid string + passphrase string + algorithm string + tokenURL string + scopes []string +} + +// JWTM2MCredentials exposes the private-key assertion inputs so the kernel +// backend can drive the kernel's own JWT client-assertion flow. It structurally +// satisfies the JWTM2MCredentialsProvider interface the kernel backend asserts +// (defined in internal/backend/kernel, so the key-reading capability is not part +// of the driver's public API). +func (c *authClient) JWTM2MCredentials() (clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string) { + return c.clientID, c.keyFile, c.kid, c.passphrase, c.algorithm, c.tokenURL, c.scopes +} + +// Authenticate is unsupported on the pure-Go (Thrift) path: signing a JWT +// client assertion is done by the native kernel, not the Go driver. Use the +// kernel backend (WithUseKernel(true)) with this authenticator. +func (c *authClient) Authenticate(r *http.Request) error { + return fmt.Errorf("jwtm2m: JWT private-key M2M is only supported on the kernel backend; " + + "enable it with WithUseKernel(true)") +} diff --git a/auth/oauth/jwtm2m/jwtm2m_test.go b/auth/oauth/jwtm2m/jwtm2m_test.go new file mode 100644 index 00000000..c01b29a1 --- /dev/null +++ b/auth/oauth/jwtm2m/jwtm2m_test.go @@ -0,0 +1,45 @@ +package jwtm2m + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJWTM2MCredentials(t *testing.T) { + t.Run("forwards all fields verbatim", func(t *testing.T) { + a := NewAuthenticator( + "sp-uuid", "/keys/jwt.pem", "kid-1", "pw", "ES256", + "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + []string{"resource/.default"}, + ) + clientID, keyFile, kid, passphrase, algorithm, tokenURL, scopes := + a.(*authClient).JWTM2MCredentials() + assert.Equal(t, "sp-uuid", clientID) + assert.Equal(t, "/keys/jwt.pem", keyFile) + assert.Equal(t, "kid-1", kid) + assert.Equal(t, "pw", passphrase) + assert.Equal(t, "ES256", algorithm) + assert.Equal(t, "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", tokenURL) + assert.Equal(t, []string{"resource/.default"}, scopes) + }) + + t.Run("empty optionals stay empty", func(t *testing.T) { + a := NewAuthenticator("sp", "/k.pem", "kid", "", "", "", nil) + _, _, _, passphrase, algorithm, tokenURL, scopes := a.(*authClient).JWTM2MCredentials() + assert.Equal(t, "", passphrase) + assert.Equal(t, "", algorithm) + assert.Equal(t, "", tokenURL) + assert.Nil(t, scopes) + }) +} + +func TestAuthenticateIsKernelOnly(t *testing.T) { + // The Go (Thrift) path can't sign a JWT client assertion — signing is the + // kernel's job — so Authenticate must fail loudly pointing at the kernel. + a := NewAuthenticator("sp", "/k.pem", "kid", "", "", "", nil) + err := a.Authenticate(&http.Request{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "kernel backend") +} diff --git a/connector.go b/connector.go index 040c3271..8b486e8e 100644 --- a/connector.go +++ b/connector.go @@ -12,6 +12,7 @@ import ( "time" "github.com/databricks/databricks-sql-go/auth" + "github.com/databricks/databricks-sql-go/auth/oauth/jwtm2m" "github.com/databricks/databricks-sql-go/auth/oauth/m2m" "github.com/databricks/databricks-sql-go/auth/pat" "github.com/databricks/databricks-sql-go/auth/tokenprovider" @@ -531,6 +532,52 @@ func WithClientCredentials(clientID, clientSecret string) ConnOption { } } +// JWTPrivateKeyM2MConfig configures OAuth machine-to-machine authentication via +// a JWT private-key client assertion (RFC 7523). A struct is used (rather than +// positional args) so the many string fields can't be transposed. +// +// KERNEL BACKEND ONLY: the assertion is signed by the native kernel, so this +// requires WithUseKernel(true). ClientID, KeyFile, and Kid are required. +type JWTPrivateKeyM2MConfig struct { + // ClientID is the service principal / OAuth client id (the assertion + // issuer and subject). + ClientID string + // KeyFile is the path to the PEM-encoded private key that signs the + // assertion. + KeyFile string + // Kid is the key id written into the JWT header so the IdP can select the + // registered public key. + Kid string + // Passphrase decrypts an encrypted PKCS#8 key; leave empty for an + // unencrypted key. + Passphrase string + // Algorithm is the JWT signing algorithm (RS256/384/512, PS256/384/512, + // ES256, ES384); empty defaults to RS256. + Algorithm string + // TokenURL is the OAuth IdP token endpoint. Required when the workspace's + // OAuth authority is an external IdP (e.g. Entra ID for Azure Databricks), + // since Databricks-native OIDC does not advertise the private_key_jwt + // method; empty falls back to the kernel's OIDC discovery. + TokenURL string + // Scopes overrides the requested OAuth scopes; empty uses the kernel + // default (all-apis). + Scopes []string +} + +// WithJWTPrivateKeyM2M sets up OAuth M2M authentication using a JWT private-key +// client assertion. See JWTPrivateKeyM2MConfig. Requires the kernel backend +// (WithUseKernel(true)); on the default (Thrift) backend a connection built with +// this authenticator fails at authenticate time with a clear kernel-only error. +func WithJWTPrivateKeyM2M(cfg JWTPrivateKeyM2MConfig) ConnOption { + return func(c *config.Config) { + if cfg.ClientID != "" && cfg.KeyFile != "" && cfg.Kid != "" { + c.Authenticator = jwtm2m.NewAuthenticator( + cfg.ClientID, cfg.KeyFile, cfg.Kid, cfg.Passphrase, cfg.Algorithm, cfg.TokenURL, cfg.Scopes, + ) + } + } +} + // WithTokenProvider sets up authentication using a custom token provider func WithTokenProvider(provider tokenprovider.TokenProvider) ConnOption { return func(c *config.Config) { diff --git a/doc.go b/doc.go index 266deff0..7507b193 100644 --- a/doc.go +++ b/doc.go @@ -92,6 +92,8 @@ Supported functional options include: - WithMaxDownloadThreads ( int). Sets up the max number of concurrent workers for cloud fetch. Default is 10. Optional - WithAuthenticator ( auth.Authenticator). Sets up authentication. Required if neither access token or client credentials are provided. - WithClientCredentials( string, string). Sets up Oauth M2M authentication. + - WithJWTPrivateKeyM2M( JWTPrivateKeyM2MConfig). Sets up OAuth M2M authentication with a JWT private-key client assertion (RFC 7523) instead of a client secret. Kernel backend only (requires WithUseKernel(true)); the kernel signs the assertion. See the kernel-backend section. Optional + - WithUseKernel( bool). Routes execution through the SEA-via-kernel backend instead of Thrift. Requires a build with -tags databricks_kernel (CGO_ENABLED=1); the default build returns a clear error. Default is false. See the kernel-backend section below. Optional - WithWarehouseID( string). The bare SQL warehouse id used by the kernel backend in preference to the http path; ignored by the Thrift backend. Optional @@ -209,8 +211,9 @@ defers to RUST_LOG. Filter on the target databricks::sql::kernel (note the colon # kernel logs plus its HTTP stack: DBSQL_KERNEL_DEBUG=1 RUST_LOG=debug ./your_app 2>&1 -Supported on the kernel backend: PAT and OAuth (M2M via WithClientCredentials, U2M -via the authType=oauthU2M DSN param); reading scalar, nested, and complex-typed +Supported on the kernel backend: PAT and OAuth (M2M via WithClientCredentials, JWT +private-key M2M via WithJWTPrivateKeyM2M, U2M via the authType=oauthU2M DSN param); +reading scalar, nested, and complex-typed results (CloudFetch is transparent); bound query parameters (positional and named); context cancellation during execute; the initial namespace (WithInitialNamespace, applied post-connect via USE CATALOG / USE SCHEMA); metric-view metadata diff --git a/internal/backend/kernel/auth.go b/internal/backend/kernel/auth.go index ddef6510..a6b43e1a 100644 --- a/internal/backend/kernel/auth.go +++ b/internal/backend/kernel/auth.go @@ -10,9 +10,10 @@ package kernel type AuthMode int const ( - AuthPAT AuthMode = iota // personal access token - AuthM2M // OAuth client-credentials (client id + secret) - AuthU2M // OAuth user-to-machine (browser/PKCE; kernel-owned flow) + AuthPAT AuthMode = iota // personal access token + AuthM2M // OAuth client-credentials (client id + secret) + AuthU2M // OAuth user-to-machine (browser/PKCE; kernel-owned flow) + AuthJWTM2M // OAuth client-credentials via a JWT private-key client assertion ) // Auth is the resolved auth descriptor for a kernel connection. Only the fields @@ -29,10 +30,18 @@ const ( type Auth struct { Mode AuthMode Token string // PAT - ClientID string // M2M + U2M (U2M: the cloud-inferred Go client id) + ClientID string // M2M + U2M + JWT M2M (U2M: the cloud-inferred Go client id) ClientSecret string // M2M - Scopes []string // U2M — Thrift-parity scopes from oauth.GetScopes; nil → kernel default + Scopes []string // U2M / JWT M2M — nil → kernel default RedirectPort uint16 // U2M — no user option today; 0 → kernel default port (8020) + + // JWT private-key M2M (Mode == AuthJWTM2M). The kernel signs a short-lived + // client assertion with the private key instead of sending a secret. + JWTKeyFile string // path to the PEM private key (required) + JWTKid string // key id written into the JWT header (required) + JWTPassphrase string // passphrase for an encrypted PKCS#8 key ("" → unencrypted) + JWTAlgorithm string // signing algorithm ("" → kernel default RS256) + TokenURL string // OAuth IdP token endpoint ("" → kernel OIDC discovery) } // M2MCredentialsProvider is implemented by the OAuth M2M authenticator to expose @@ -78,3 +87,18 @@ type U2MCredentialsProvider interface { // U2MClientID returns the OAuth client id for the U2M browser flow. U2MClientID() string } + +// JWTM2MCredentialsProvider is implemented by the JWT private-key M2M +// authenticator to expose the assertion-signing inputs. The kernel backend reads +// these to drive the kernel's own JWT client-assertion flow (the kernel signs the +// assertion and owns the token exchange), rather than using the authenticator's +// Authenticate method — which is unsupported on the pure-Go Thrift path anyway. +// Internal for the same reason as M2MCredentialsProvider (the key-reading +// capability is not part of the driver's public API); satisfied structurally by +// the unexported jwtm2m authenticator. +type JWTM2MCredentialsProvider interface { + // JWTM2MCredentials returns the client id, private-key file path, key id, + // passphrase (may be ""), algorithm (may be ""), token URL (may be ""), and + // scopes (may be nil). + JWTM2MCredentials() (clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string) +} diff --git a/internal/backend/kernel/backend.go b/internal/backend/kernel/backend.go index 500144f6..33efa755 100644 --- a/internal/backend/kernel/backend.go +++ b/internal/backend/kernel/backend.go @@ -411,6 +411,33 @@ func (k *KernelBackend) setAuth(cfg *C.KernelSessionConfig) error { }); err != nil { return fmt.Errorf("kernel: set_auth_u2m: %w", toConnError(err)) } + case AuthJWTM2M: + // JWT private-key client assertion (RFC 7523). client_id / jwt_key_file / + // jwt_kid are required; passphrase / algorithm / scopes / token_url are + // optional — NULL lets the kernel fill its defaults (unencrypted key, RS256, + // all-apis, OIDC discovery). token_url points the grant at the workspace's + // OAuth IdP (e.g. Entra ID) when Databricks-native OIDC can't serve it. + clientID := newCStr(k.cfg.Auth.ClientID) + defer clientID.free() + keyFile := newCStr(k.cfg.Auth.JWTKeyFile) + defer keyFile.free() + kid := newCStr(k.cfg.Auth.JWTKid) + defer kid.free() + passphrase := newCStrOrNull(k.cfg.Auth.JWTPassphrase) + defer passphrase.free() + algorithm := newCStrOrNull(k.cfg.Auth.JWTAlgorithm) + defer algorithm.free() + scopes := newCStrOrNull(joinScopes(k.cfg.Auth.Scopes)) + defer scopes.free() + tokenURL := newCStrOrNull(k.cfg.Auth.TokenURL) + defer tokenURL.free() + if err := call(func() C.KernelStatusCode { + return C.kernel_session_config_set_auth_m2m_jwt( + cfg, clientID.c, keyFile.c, kid.c, passphrase.c, algorithm.c, scopes.c, tokenURL.c, + ) + }); err != nil { + return fmt.Errorf("kernel: set_auth_m2m_jwt: %w", toConnError(err)) + } default: // AuthPAT tok := newCStr(k.cfg.Auth.Token) defer tok.free() diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index 9a175c7d..b6c2cbaf 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -42,6 +42,22 @@ func TestSetAuthByMode(t *testing.T) { // scopes / port 0 must pass NULL / 0 so the kernel applies its own defaults // (exercises newCStrOrNull). {"U2M defaults", Auth{Mode: AuthU2M}}, + // JWT M2M full: all optional args populated, exercising the 8-arg + // set_auth_m2m_jwt marshalling (client_id/key_file/kid required, plus + // passphrase/algorithm/scopes/token_url). + {"JWT M2M full", Auth{ + Mode: AuthJWTM2M, + ClientID: "sp-uuid", + JWTKeyFile: "/keys/jwt.pem", + JWTKid: "kid-1", + JWTPassphrase: "pw", + JWTAlgorithm: "ES256", + TokenURL: "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", + Scopes: []string{"resource/.default"}, + }}, + // JWT M2M with only the required fields: passphrase/algorithm/scopes/token_url + // pass NULL so the kernel applies its defaults (exercises newCStrOrNull). + {"JWT M2M defaults", Auth{Mode: AuthJWTM2M, ClientID: "sp", JWTKeyFile: "/k.pem", JWTKid: "kid"}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -89,7 +105,7 @@ func TestSetProxy(t *testing.T) { }{ {"url only", Config{ProxyURL: "http://proxy:3128"}}, {"url + credentials", Config{ProxyURL: "http://proxy:3128", ProxyUsername: "u", ProxyPassword: "p"}}, - {"url + bypass", Config{ProxyURL: "http://proxy:3128", ProxyBypassHosts: "localhost,*.internal"}}, //nolint:gosec // G101: test literals, not real credentials + {"url + bypass", Config{ProxyURL: "http://proxy:3128", ProxyBypassHosts: "localhost,*.internal"}}, //nolint:gosec // G101: test literals, not real credentials {"all fields", Config{ProxyURL: "http://proxy:3128", ProxyUsername: "u", ProxyPassword: "p", ProxyBypassHosts: "localhost,*.internal"}}, //nolint:gosec // G101: test literals, not real credentials {"none (no-op)", Config{}}, } @@ -104,9 +120,10 @@ func TestSetProxy(t *testing.T) { // TestSetRetry exercises the real kernel_session_config_set_retry_config cgo setter // via the trySetRetry seam: a valid range succeeds (incl. the disable form, -// MaxRetries=0, and a non-zero overall budget), and a degenerate range (min=0 or -// max +// DATABRICKS_JWT_CLIENT_ID service principal / OAuth client id +// DATABRICKS_JWT_KEY_FILE path to the PEM private key +// DATABRICKS_JWT_KID key id (Entra: the cert x5t thumbprint) +// DATABRICKS_JWT_TOKEN_URL IdP token endpoint (optional; OIDC discovery otherwise) +// DATABRICKS_JWT_SCOPES space-separated scope override (optional) +func TestKernelE2EJWTM2MSelect1(t *testing.T) { + host := os.Getenv("DATABRICKS_HOST") + httpPath := os.Getenv("DATABRICKS_HTTP_PATH") + clientID := os.Getenv("DATABRICKS_JWT_CLIENT_ID") + keyFile := os.Getenv("DATABRICKS_JWT_KEY_FILE") + kid := os.Getenv("DATABRICKS_JWT_KID") + if host == "" || httpPath == "" || clientID == "" || keyFile == "" || kid == "" { + t.Skip("JWT M2M creds unset (DATABRICKS_HOST / DATABRICKS_HTTP_PATH / " + + "DATABRICKS_JWT_CLIENT_ID / DATABRICKS_JWT_KEY_FILE / DATABRICKS_JWT_KID)") + } + + var scopes []string + if s := os.Getenv("DATABRICKS_JWT_SCOPES"); s != "" { + scopes = strings.Fields(s) + } + + connector, err := NewConnector( + WithServerHostname(host), + WithHTTPPath(httpPath), + WithJWTPrivateKeyM2M(JWTPrivateKeyM2MConfig{ + ClientID: clientID, + KeyFile: keyFile, + Kid: kid, + Passphrase: os.Getenv("DATABRICKS_JWT_PASSPHRASE"), + Algorithm: os.Getenv("DATABRICKS_JWT_ALGORITHM"), + TokenURL: os.Getenv("DATABRICKS_JWT_TOKEN_URL"), + Scopes: scopes, + }), + WithUseKernel(true), + ) + if err != nil { + t.Fatalf("NewConnector: %v", err) + } + db := sql.OpenDB(connector) + defer db.Close() + + var got int + if err := db.QueryRowContext(context.Background(), "SELECT 1").Scan(&got); err != nil { + t.Fatalf("SELECT 1 via kernel JWT M2M: %v", err) + } + if got != 1 { + t.Fatalf("SELECT 1 = %d, want 1", got) + } +} From 2ab4b65df284b92c09ceff1ecee79cef8e8bf7a5 Mon Sep 17 00:00:00 2001 From: Rahul Singhal Date: Thu, 20 Aug 2026 02:58:44 +0000 Subject: [PATCH 2/4] style: gofmt doc.go + suppress gosec G101 on JWT test literals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address golangci-lint failures on #447: - gofmt doc.go after the WithJWTPrivateKeyM2M option-list addition. - //nolint:gosec G101 on the fakeJWTM2MAuth test literal (a fake passphrase in a unit test, not a real credential) — same convention the existing proxy tests use. Signed-off-by: Rahul Singhal --- doc.go | 14 ++++++++++++++ kernel_config_test.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc.go b/doc.go index 7507b193..5044f332 100644 --- a/doc.go +++ b/doc.go @@ -80,21 +80,35 @@ Use sql.OpenDB() to create a database handle via a new connector object created Supported functional options include: - WithServerHostname( string): Sets up the server hostname. The hostname can be prefixed with "http:" or "https:" to specify a protocol to use. Mandatory + - WithPort( int): Sets up the server port. Mandatory + - WithAccessToken( string): Sets up the Personal Access Token. Mandatory + - WithHTTPPath( string): Sets up the endpoint to the warehouse. Mandatory + - WithInitialNamespace( string, string): Sets up the catalog and schema name in the session. Optional + - WithMaxRows( int): Sets up the max rows fetched per request. Default is 100000. Optional + - WithSessionParams( map[string]string): Sets up session parameters including "timezone" and "ansi_mode". Optional + - WithTimeout( Duration). Adds timeout (in time.Duration) for the server query execution. Default is no timeout. Optional + - WithUserAgentEntry( string). Used to identify partners. Optional + - WithCloudFetch (bool). Used to enable cloud fetch for the query execution. Default is true. Optional + - WithMaxDownloadThreads ( int). Sets up the max number of concurrent workers for cloud fetch. Default is 10. Optional + - WithAuthenticator ( auth.Authenticator). Sets up authentication. Required if neither access token or client credentials are provided. + - WithClientCredentials( string, string). Sets up Oauth M2M authentication. + - WithJWTPrivateKeyM2M( JWTPrivateKeyM2MConfig). Sets up OAuth M2M authentication with a JWT private-key client assertion (RFC 7523) instead of a client secret. Kernel backend only (requires WithUseKernel(true)); the kernel signs the assertion. See the kernel-backend section. Optional - WithUseKernel( bool). Routes execution through the SEA-via-kernel backend instead of Thrift. Requires a build with -tags databricks_kernel (CGO_ENABLED=1); the default build returns a clear error. Default is false. See the kernel-backend section below. Optional + - WithWarehouseID( string). The bare SQL warehouse id used by the kernel backend in preference to the http path; ignored by the Thrift backend. Optional # Query cancellation and timeout diff --git a/kernel_config_test.go b/kernel_config_test.go index 6335725b..f7a36fad 100644 --- a/kernel_config_test.go +++ b/kernel_config_test.go @@ -211,7 +211,7 @@ func TestValidateKernelConfig(t *testing.T) { // reads its inputs via the auth.JWTM2MCredentialsProvider interface. Unlike // shared-secret M2M, the kernel's JWT setter carries scopes + token_url, so // custom values forward as-is (no rejection). - c.Authenticator = fakeJWTM2MAuth{ + c.Authenticator = fakeJWTM2MAuth{ //nolint:gosec // G101: test literals, not real credentials id: "sp-uuid", keyFile: "/keys/jwt.pem", kid: "kid-1", From 268fb1e84df7d762e747f5ff27a8126689562bba Mon Sep 17 00:00:00 2001 From: Rahul Singhal Date: Thu, 20 Aug 2026 20:04:33 +0000 Subject: [PATCH 3/4] fix(kernel): gate JWT M2M public surface behind kernel build tag Addresses PR #447 review feedback: - Gate the public JWT private-key M2M surface behind `cgo && databricks_kernel`. Move WithJWTPrivateKeyM2M and JWTPrivateKeyM2MConfig out of connector.go into a new tagged connector_kernel_jwt.go, and tag the jwtm2m package itself. The pure-Go Thrift path has no JWT-signing implementation, so a default (Thrift-only) build no longer exposes an auth option it cannot honor (Eric's divergence concern). The internal JWTM2MCredentialsProvider interface stays untagged so resolveKernelAuth still compiles in the default build. - Skip telemetry on the kernel JWT M2M path: its Authenticate always returns a kernel-only error, so telemetry/feature-flag HTTP calls would fail every request and burn round-trips at connect. Extend the existing U2M telemetry-skip to cover it (bot F2). - Fix the godoc option list: restore the contiguous bullet style (a blank line between items terminates the godoc list) and add the JWT bullet inline (bot F1). - Update the stale supported-auth enumerations: resolveKernelAuth doc, validateKernelConfig docstring, and the default-case rejection message now include JWT private-key M2M (bot F2, first review). Co-authored-by: Isaac Signed-off-by: Rahul Singhal --- auth/oauth/jwtm2m/jwtm2m.go | 2 + auth/oauth/jwtm2m/jwtm2m_test.go | 2 + connector.go | 66 +++++++++----------------------- connector_kernel_jwt.go | 59 ++++++++++++++++++++++++++++ doc.go | 17 +------- kernel_config.go | 13 ++++--- 6 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 connector_kernel_jwt.go diff --git a/auth/oauth/jwtm2m/jwtm2m.go b/auth/oauth/jwtm2m/jwtm2m.go index bc71f2f0..d0999af9 100644 --- a/auth/oauth/jwtm2m/jwtm2m.go +++ b/auth/oauth/jwtm2m/jwtm2m.go @@ -1,3 +1,5 @@ +//go:build cgo && databricks_kernel + // Package jwtm2m provides an OAuth machine-to-machine authenticator that // authenticates with a JWT private-key client assertion (RFC 7523) instead of // a client secret. diff --git a/auth/oauth/jwtm2m/jwtm2m_test.go b/auth/oauth/jwtm2m/jwtm2m_test.go index c01b29a1..c123a55d 100644 --- a/auth/oauth/jwtm2m/jwtm2m_test.go +++ b/auth/oauth/jwtm2m/jwtm2m_test.go @@ -1,3 +1,5 @@ +//go:build cgo && databricks_kernel + package jwtm2m import ( diff --git a/connector.go b/connector.go index 8b486e8e..f96ff1b3 100644 --- a/connector.go +++ b/connector.go @@ -12,7 +12,6 @@ import ( "time" "github.com/databricks/databricks-sql-go/auth" - "github.com/databricks/databricks-sql-go/auth/oauth/jwtm2m" "github.com/databricks/databricks-sql-go/auth/oauth/m2m" "github.com/databricks/databricks-sql-go/auth/pat" "github.com/databricks/databricks-sql-go/auth/tokenprovider" @@ -39,6 +38,19 @@ type interactiveU2MAuthenticator interface { U2MClientID() string } +// kernelOnlyAuthenticator is satisfied only by the JWT private-key M2M +// authenticator, whose Authenticate always returns a kernel-only error (the +// assertion is signed by the native kernel, not the Go path). Matches the +// JWTM2MCredentials signature the kernel backend asserts structurally. Used to +// skip telemetry on the kernel JWT M2M path: telemetry/feature-flag HTTP calls +// go through cfg.Authenticator.Authenticate, which for this authenticator would +// fail every request (burning a failing round-trip at connect). Telemetry is +// best-effort, so it's dropped here rather than made to fail — consistent with +// the U2M skip below and the Python/Node kernel paths (PECOBLR-3839). +type kernelOnlyAuthenticator interface { + JWTM2MCredentials() (clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string) +} + // Connect returns a connection to the Databricks database from a connection pool. func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { defer debuglog.Track(ctx, "connector.Connect", "host=%s", c.cfg.Host)() @@ -104,6 +116,12 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { if _, isU2M := c.cfg.Authenticator.(interactiveU2MAuthenticator); isU2M { skipTelemetry = true log.Debug().Msg("telemetry skipped: kernel U2M owns the interactive auth flow") + } else if _, isKernelOnly := c.cfg.Authenticator.(kernelOnlyAuthenticator); isKernelOnly { + // JWT private-key M2M: Authenticate always errors on the Go path (the + // kernel signs the assertion), so every telemetry/feature-flag request + // would fail auth. Drop telemetry rather than burn failing round-trips. + skipTelemetry = true + log.Debug().Msg("telemetry skipped: kernel JWT M2M signs via the native kernel; Go-path auth is kernel-only") } } @@ -532,52 +550,6 @@ func WithClientCredentials(clientID, clientSecret string) ConnOption { } } -// JWTPrivateKeyM2MConfig configures OAuth machine-to-machine authentication via -// a JWT private-key client assertion (RFC 7523). A struct is used (rather than -// positional args) so the many string fields can't be transposed. -// -// KERNEL BACKEND ONLY: the assertion is signed by the native kernel, so this -// requires WithUseKernel(true). ClientID, KeyFile, and Kid are required. -type JWTPrivateKeyM2MConfig struct { - // ClientID is the service principal / OAuth client id (the assertion - // issuer and subject). - ClientID string - // KeyFile is the path to the PEM-encoded private key that signs the - // assertion. - KeyFile string - // Kid is the key id written into the JWT header so the IdP can select the - // registered public key. - Kid string - // Passphrase decrypts an encrypted PKCS#8 key; leave empty for an - // unencrypted key. - Passphrase string - // Algorithm is the JWT signing algorithm (RS256/384/512, PS256/384/512, - // ES256, ES384); empty defaults to RS256. - Algorithm string - // TokenURL is the OAuth IdP token endpoint. Required when the workspace's - // OAuth authority is an external IdP (e.g. Entra ID for Azure Databricks), - // since Databricks-native OIDC does not advertise the private_key_jwt - // method; empty falls back to the kernel's OIDC discovery. - TokenURL string - // Scopes overrides the requested OAuth scopes; empty uses the kernel - // default (all-apis). - Scopes []string -} - -// WithJWTPrivateKeyM2M sets up OAuth M2M authentication using a JWT private-key -// client assertion. See JWTPrivateKeyM2MConfig. Requires the kernel backend -// (WithUseKernel(true)); on the default (Thrift) backend a connection built with -// this authenticator fails at authenticate time with a clear kernel-only error. -func WithJWTPrivateKeyM2M(cfg JWTPrivateKeyM2MConfig) ConnOption { - return func(c *config.Config) { - if cfg.ClientID != "" && cfg.KeyFile != "" && cfg.Kid != "" { - c.Authenticator = jwtm2m.NewAuthenticator( - cfg.ClientID, cfg.KeyFile, cfg.Kid, cfg.Passphrase, cfg.Algorithm, cfg.TokenURL, cfg.Scopes, - ) - } - } -} - // WithTokenProvider sets up authentication using a custom token provider func WithTokenProvider(provider tokenprovider.TokenProvider) ConnOption { return func(c *config.Config) { diff --git a/connector_kernel_jwt.go b/connector_kernel_jwt.go new file mode 100644 index 00000000..c16e1060 --- /dev/null +++ b/connector_kernel_jwt.go @@ -0,0 +1,59 @@ +//go:build cgo && databricks_kernel + +package dbsql + +import ( + "github.com/databricks/databricks-sql-go/auth/oauth/jwtm2m" + "github.com/databricks/databricks-sql-go/internal/config" +) + +// JWTPrivateKeyM2MConfig configures OAuth machine-to-machine authentication via +// a JWT private-key client assertion (RFC 7523). A struct is used (rather than +// positional args) so the many string fields can't be transposed. +// +// KERNEL BACKEND ONLY: the assertion is signed by the native kernel, so both +// this type and WithJWTPrivateKeyM2M exist only in a build compiled with the +// kernel backend (`cgo && databricks_kernel`). A default (Thrift-only) build +// does not expose them, so a Thrift user can't configure an auth mode the +// Thrift path cannot honor. +type JWTPrivateKeyM2MConfig struct { + // ClientID is the service principal / OAuth client id (the assertion + // issuer and subject). + ClientID string + // KeyFile is the path to the PEM-encoded private key that signs the + // assertion. + KeyFile string + // Kid is the key id written into the JWT header so the IdP can select the + // registered public key. + Kid string + // Passphrase decrypts an encrypted PKCS#8 key; leave empty for an + // unencrypted key. + Passphrase string + // Algorithm is the JWT signing algorithm (RS256/384/512, PS256/384/512, + // ES256, ES384); empty defaults to RS256. + Algorithm string + // TokenURL is the OAuth IdP token endpoint. Required when the workspace's + // OAuth authority is an external IdP (e.g. Entra ID for Azure Databricks), + // since Databricks-native OIDC does not advertise the private_key_jwt + // method; empty falls back to the kernel's OIDC discovery. + TokenURL string + // Scopes overrides the requested OAuth scopes; empty uses the kernel + // default (all-apis). + Scopes []string +} + +// WithJWTPrivateKeyM2M sets up OAuth M2M authentication using a JWT private-key +// client assertion. See JWTPrivateKeyM2MConfig. Requires the kernel backend +// (WithUseKernel(true)); the kernel signs the assertion. This option is only +// compiled into a kernel-enabled build (`cgo && databricks_kernel`) — the +// pure-Go Thrift path has no JWT-signing implementation, so exposing it there +// would only add a divergent, non-functional surface. +func WithJWTPrivateKeyM2M(cfg JWTPrivateKeyM2MConfig) ConnOption { + return func(c *config.Config) { + if cfg.ClientID != "" && cfg.KeyFile != "" && cfg.Kid != "" { + c.Authenticator = jwtm2m.NewAuthenticator( + cfg.ClientID, cfg.KeyFile, cfg.Kid, cfg.Passphrase, cfg.Algorithm, cfg.TokenURL, cfg.Scopes, + ) + } + } +} diff --git a/doc.go b/doc.go index 5044f332..8753a9cd 100644 --- a/doc.go +++ b/doc.go @@ -80,35 +80,20 @@ Use sql.OpenDB() to create a database handle via a new connector object created Supported functional options include: - WithServerHostname( string): Sets up the server hostname. The hostname can be prefixed with "http:" or "https:" to specify a protocol to use. Mandatory - - WithPort( int): Sets up the server port. Mandatory - - WithAccessToken( string): Sets up the Personal Access Token. Mandatory - - WithHTTPPath( string): Sets up the endpoint to the warehouse. Mandatory - - WithInitialNamespace( string, string): Sets up the catalog and schema name in the session. Optional - - WithMaxRows( int): Sets up the max rows fetched per request. Default is 100000. Optional - - WithSessionParams( map[string]string): Sets up session parameters including "timezone" and "ansi_mode". Optional - - WithTimeout( Duration). Adds timeout (in time.Duration) for the server query execution. Default is no timeout. Optional - - WithUserAgentEntry( string). Used to identify partners. Optional - - WithCloudFetch (bool). Used to enable cloud fetch for the query execution. Default is true. Optional - - WithMaxDownloadThreads ( int). Sets up the max number of concurrent workers for cloud fetch. Default is 10. Optional - - WithAuthenticator ( auth.Authenticator). Sets up authentication. Required if neither access token or client credentials are provided. - - WithClientCredentials( string, string). Sets up Oauth M2M authentication. - - - WithJWTPrivateKeyM2M( JWTPrivateKeyM2MConfig). Sets up OAuth M2M authentication with a JWT private-key client assertion (RFC 7523) instead of a client secret. Kernel backend only (requires WithUseKernel(true)); the kernel signs the assertion. See the kernel-backend section. Optional - + - WithJWTPrivateKeyM2M( JWTPrivateKeyM2MConfig). Sets up OAuth M2M authentication with a JWT private-key client assertion (RFC 7523) instead of a client secret. Kernel backend only (requires WithUseKernel(true) and a build with -tags databricks_kernel); the kernel signs the assertion. See the kernel-backend section. Optional - WithUseKernel( bool). Routes execution through the SEA-via-kernel backend instead of Thrift. Requires a build with -tags databricks_kernel (CGO_ENABLED=1); the default build returns a clear error. Default is false. See the kernel-backend section below. Optional - - WithWarehouseID( string). The bare SQL warehouse id used by the kernel backend in preference to the http path; ignored by the Thrift backend. Optional # Query cancellation and timeout diff --git a/kernel_config.go b/kernel_config.go index 1e85f05c..8b5c3a98 100644 --- a/kernel_config.go +++ b/kernel_config.go @@ -26,8 +26,8 @@ import ( // validateKernelConfig enforces the kernel backend's "nothing silently ignored" // contract: it rejects every option the kernel path can't yet honor with a clear // error (rather than dropping it, which would behave differently than Thrift) and -// resolves the kernel.Auth descriptor the kernel authenticates with (PAT, or OAuth -// M2M/U2M). Options it does NOT reject are either forwarded by newKernelBackend or +// resolves the kernel.Auth descriptor the kernel authenticates with (PAT, OAuth +// M2M/U2M, or JWT private-key M2M). Options it does NOT reject are either forwarded by newKernelBackend or // intentionally accepted-but-inert (documented in doc.go and asserted by // TestKernelConfigFieldsClassified). It returns kernel.Auth directly (no dbsql-side // duplicate) — kernel's auth types are in an untagged file, so this untagged, @@ -288,8 +288,10 @@ func resolveKernelProxy(cfg *config.Config, kc *kernel.Config) { // single source of truth for auth, so the last WithX option applied wins for both // backends (matching Thrift's last-writer-wins on cfg.Authenticator). The M2M/U2M // authenticator types are unexported, so it asserts the small -// kernel.M2MCredentialsProvider / kernel.U2MCredentialsProvider interfaces they -// satisfy structurally: +// kernel.JWTM2MCredentialsProvider / kernel.M2MCredentialsProvider / +// kernel.U2MCredentialsProvider interfaces they satisfy structurally: +// - implements JWTM2MCredentialsProvider → JWT private-key M2M (RFC 7523 client +// assertion; kernel signs it) // - implements M2MCredentialsProvider → M2M (client id + secret) // - implements U2MCredentialsProvider → U2M (browser/PKCE; kernel-owned flow) // - PAT / nil / noop → PAT (from AccessToken or a *pat.PATAuth) @@ -363,7 +365,8 @@ func resolveKernelAuth(cfg *config.Config) (kernel.Auth, error) { // wrapped — a missing token is misconfiguration to fix, not a feature the // kernel can't honor.) return kernel.Auth{}, fmt.Errorf("databricks: this authenticator is %w; "+ - "PAT (WithAccessToken) and OAuth M2M/U2M (WithClientCredentials / authType) are supported, but "+ + "PAT (WithAccessToken), OAuth M2M/U2M (WithClientCredentials / authType), and JWT "+ + "private-key M2M (WithJWTPrivateKeyM2M) are supported, but "+ "token-provider, external/static, and federated authenticators are not — "+ "use one of those or the default (Thrift) backend", dbsqlerr.ErrNotSupportedByKernel) } From 47ea68abdae2b02551cd2ba8e5cb72355d0cea35 Mon Sep 17 00:00:00 2001 From: Rahul Singhal Date: Thu, 20 Aug 2026 20:56:47 +0000 Subject: [PATCH 4/4] fix(kernel): warn on partial JWT M2M config; document all JWT e2e env vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR #447 review (peco-review-bot, 1 Low + a doc nit): - WithJWTPrivateKeyM2M now warns when given a PARTIAL config (some of the required ClientID/KeyFile/Kid set, others blank — e.g. a typo or unset env var). A fully-empty config stays a silent no-op (consistent with WithClientCredentials/WithAccessToken), but a partial one would otherwise install no authenticator and later surface as a misleading "requires a personal access token" error, so the misconfiguration is now visible. - Document DATABRICKS_JWT_PASSPHRASE and DATABRICKS_JWT_ALGORITHM in the TestKernelE2EJWTM2MSelect1 env-var block (the test body already reads them). Adds a kernel-tagged unit test for full/empty/partial WithJWTPrivateKeyM2M. Co-authored-by: Isaac Signed-off-by: Rahul Singhal --- connector_kernel_jwt.go | 14 +++++++++++ connector_kernel_jwt_test.go | 46 ++++++++++++++++++++++++++++++++++++ kernel_jwt_e2e_test.go | 2 ++ 3 files changed, 62 insertions(+) create mode 100644 connector_kernel_jwt_test.go diff --git a/connector_kernel_jwt.go b/connector_kernel_jwt.go index c16e1060..df14e760 100644 --- a/connector_kernel_jwt.go +++ b/connector_kernel_jwt.go @@ -5,6 +5,7 @@ package dbsql import ( "github.com/databricks/databricks-sql-go/auth/oauth/jwtm2m" "github.com/databricks/databricks-sql-go/internal/config" + "github.com/databricks/databricks-sql-go/logger" ) // JWTPrivateKeyM2MConfig configures OAuth machine-to-machine authentication via @@ -54,6 +55,19 @@ func WithJWTPrivateKeyM2M(cfg JWTPrivateKeyM2MConfig) ConnOption { c.Authenticator = jwtm2m.NewAuthenticator( cfg.ClientID, cfg.KeyFile, cfg.Kid, cfg.Passphrase, cfg.Algorithm, cfg.TokenURL, cfg.Scopes, ) + return + } + // Consistent with WithClientCredentials/WithAccessToken, a fully-empty + // config is a no-op (the caller simply didn't pick this option). But a + // PARTIAL config (some required fields set, others blank — e.g. a typo or + // an unset env var) is almost certainly a mistake: no authenticator gets + // installed, so connect later fails with a misleading "requires a personal + // access token" (or authenticates as PAT if WithAccessToken was also set). + // Warn so the misconfiguration isn't wholly invisible, rather than + // silently dropping it. + if cfg.ClientID != "" || cfg.KeyFile != "" || cfg.Kid != "" { + logger.Warn().Msg("WithJWTPrivateKeyM2M: incomplete config; ClientID, KeyFile, and Kid are all " + + "required. No JWT authenticator was installed — the connection will not use JWT private-key M2M.") } } } diff --git a/connector_kernel_jwt_test.go b/connector_kernel_jwt_test.go new file mode 100644 index 00000000..5fea1837 --- /dev/null +++ b/connector_kernel_jwt_test.go @@ -0,0 +1,46 @@ +//go:build cgo && databricks_kernel + +package dbsql + +import ( + "testing" + + "github.com/databricks/databricks-sql-go/internal/config" +) + +func TestWithJWTPrivateKeyM2M(t *testing.T) { + apply := func(cfg JWTPrivateKeyM2MConfig) *config.Config { + c := &config.Config{} + WithJWTPrivateKeyM2M(cfg)(c) + return c + } + + t.Run("full config installs the JWT authenticator", func(t *testing.T) { + c := apply(JWTPrivateKeyM2MConfig{ClientID: "sp", KeyFile: "/k.pem", Kid: "kid-1"}) + if c.Authenticator == nil { + t.Fatal("expected an authenticator to be installed for a complete JWT config") + } + // It must satisfy the interface the kernel backend asserts to select JWT M2M. + if _, ok := c.Authenticator.(interface { + JWTM2MCredentials() (string, string, string, string, string, string, []string) + }); !ok { + t.Fatalf("installed authenticator %T does not expose JWTM2MCredentials", c.Authenticator) + } + }) + + t.Run("empty config is a silent no-op", func(t *testing.T) { + c := apply(JWTPrivateKeyM2MConfig{}) + if c.Authenticator != nil { + t.Fatalf("expected no authenticator for an empty config, got %T", c.Authenticator) + } + }) + + t.Run("partial config installs no authenticator", func(t *testing.T) { + // ClientID + KeyFile but no Kid: the required trio is incomplete, so no + // authenticator is installed (and WithJWTPrivateKeyM2M warns). + c := apply(JWTPrivateKeyM2MConfig{ClientID: "sp", KeyFile: "/k.pem"}) + if c.Authenticator != nil { + t.Fatalf("expected no authenticator for a partial config, got %T", c.Authenticator) + } + }) +} diff --git a/kernel_jwt_e2e_test.go b/kernel_jwt_e2e_test.go index 0dca9472..2b7cf82a 100644 --- a/kernel_jwt_e2e_test.go +++ b/kernel_jwt_e2e_test.go @@ -24,6 +24,8 @@ import ( // DATABRICKS_JWT_CLIENT_ID service principal / OAuth client id // DATABRICKS_JWT_KEY_FILE path to the PEM private key // DATABRICKS_JWT_KID key id (Entra: the cert x5t thumbprint) +// DATABRICKS_JWT_PASSPHRASE passphrase for an encrypted PKCS#8 key (optional) +// DATABRICKS_JWT_ALGORITHM JWT signing algorithm (optional; default RS256) // DATABRICKS_JWT_TOKEN_URL IdP token endpoint (optional; OIDC discovery otherwise) // DATABRICKS_JWT_SCOPES space-separated scope override (optional) func TestKernelE2EJWTM2MSelect1(t *testing.T) {