feat(kernel): JWT private-key M2M auth via WithJWTPrivateKeyM2M

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Release History
 
 ## Unreleased
+- Kernel backend (`WithUseKernel(true)`): OAuth **M2M with a JWT private-key client assertion** (RFC 7523) is now supported via `WithJWTPrivateKeyM2M(JWTPrivateKeyM2MConfig{...})`. The kernel signs a short-lived assertion with the private key instead of sending a client secret and owns the token lifecycle. `ClientID`, `KeyFile`, and `Kid` are required; `Passphrase` (encrypted PKCS#8 key), `Algorithm` (default `RS256`), `TokenURL` (the OAuth IdP token endpoint — required for an external-IdP-backed workspace such as Entra ID, since Databricks-native OIDC does not advertise the `private_key_jwt` method), and `Scopes` are optional. Kernel-backend only; on the default (Thrift) backend a connection built with this authenticator fails at authenticate time with a clear kernel-only error. Verified end-to-end against an Azure Databricks warehouse via Entra ID. Also bumps the pinned `KERNEL_REV` to a build that includes the JWT C-ABI setter. DSN-string configuration (`authType=...`) for JWT M2M is deferred to a follow-up; use the programmatic option.
 - Improve telemetry error reporting: driver failures are now categorized by cause instead of reported as a generic error (databricks/databricks-sql-go#414, #415, #417, #419, #424)
 
 ## v1.14.0 (2026-07-13)

KERNEL_REV

@@ -1 +1 @@
-9ac3f3d3f3e804d52d8890e890d9f8a8a617ec93
+ef2c8bedb0345076d3147e05c35362778ac4e130

auth/oauth/jwtm2m/jwtm2m.go

@@ -0,0 +1,65 @@
+//go:build cgo && databricks_kernel
+
+// Package jwtm2m provides an OAuth machine-to-machine authenticator that
+// authenticates with a JWT private-key client assertion (RFC 7523) instead of
+// a client secret.
+//
+// This authenticator is KERNEL-BACKEND ONLY. The JWT is signed by the native
+// kernel (which owns the assertion signing + token lifecycle); the pure-Go
+// Thrift path has no JWT-signing implementation, so Authenticate returns an
+// error directing the caller at the kernel backend. The authenticator exists
+// so cfg.Authenticator stays the single source of truth for auth on both
+// backends: the kernel selects this mode by asserting the (internal)
+// JWTM2MCredentialsProvider interface that this type satisfies structurally.
+package jwtm2m
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/databricks/databricks-sql-go/auth"
+)
+
+// NewAuthenticator builds a JWT private-key M2M authenticator. clientID,
+// keyFile, and kid are required; passphrase (for an encrypted PKCS#8 key),
+// algorithm (default RS256), tokenURL (the OAuth IdP token endpoint — required
+// for an external-IdP-backed workspace such as Entra ID), and scopes are
+// optional.
+func NewAuthenticator(clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string) auth.Authenticator {
+	return &authClient{
+		clientID:   clientID,
+		keyFile:    keyFile,
+		kid:        kid,
+		passphrase: passphrase,
+		algorithm:  algorithm,
+		tokenURL:   tokenURL,
+		scopes:     scopes,
+	}
+}
+
+type authClient struct {
+	clientID   string
+	keyFile    string
+	kid        string
+	passphrase string
+	algorithm  string
+	tokenURL   string
+	scopes     []string
+}
+
+// JWTM2MCredentials exposes the private-key assertion inputs so the kernel
+// backend can drive the kernel's own JWT client-assertion flow. It structurally
+// satisfies the JWTM2MCredentialsProvider interface the kernel backend asserts
+// (defined in internal/backend/kernel, so the key-reading capability is not part
+// of the driver's public API).
+func (c *authClient) JWTM2MCredentials() (clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string) {
+	return c.clientID, c.keyFile, c.kid, c.passphrase, c.algorithm, c.tokenURL, c.scopes
+}
+
+// Authenticate is unsupported on the pure-Go (Thrift) path: signing a JWT
+// client assertion is done by the native kernel, not the Go driver. Use the
+// kernel backend (WithUseKernel(true)) with this authenticator.
+func (c *authClient) Authenticate(r *http.Request) error {
+	return fmt.Errorf("jwtm2m: JWT private-key M2M is only supported on the kernel backend; " +
+		"enable it with WithUseKernel(true)")
+}

auth/oauth/jwtm2m/jwtm2m_test.go

@@ -0,0 +1,47 @@
+//go:build cgo && databricks_kernel
+
+package jwtm2m
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJWTM2MCredentials(t *testing.T) {
+	t.Run("forwards all fields verbatim", func(t *testing.T) {
+		a := NewAuthenticator(
+			"sp-uuid", "/keys/jwt.pem", "kid-1", "pw", "ES256",
+			"https://login.microsoftonline.com/tenant/oauth2/v2.0/token",
+			[]string{"resource/.default"},
+		)
+		clientID, keyFile, kid, passphrase, algorithm, tokenURL, scopes :=
+			a.(*authClient).JWTM2MCredentials()
+		assert.Equal(t, "sp-uuid", clientID)
+		assert.Equal(t, "/keys/jwt.pem", keyFile)
+		assert.Equal(t, "kid-1", kid)
+		assert.Equal(t, "pw", passphrase)
+		assert.Equal(t, "ES256", algorithm)
+		assert.Equal(t, "https://login.microsoftonline.com/tenant/oauth2/v2.0/token", tokenURL)
+		assert.Equal(t, []string{"resource/.default"}, scopes)
+	})
+
+	t.Run("empty optionals stay empty", func(t *testing.T) {
+		a := NewAuthenticator("sp", "/k.pem", "kid", "", "", "", nil)
+		_, _, _, passphrase, algorithm, tokenURL, scopes := a.(*authClient).JWTM2MCredentials()
+		assert.Equal(t, "", passphrase)
+		assert.Equal(t, "", algorithm)
+		assert.Equal(t, "", tokenURL)
+		assert.Nil(t, scopes)
+	})
+}
+
+func TestAuthenticateIsKernelOnly(t *testing.T) {
+	// The Go (Thrift) path can't sign a JWT client assertion — signing is the
+	// kernel's job — so Authenticate must fail loudly pointing at the kernel.
+	a := NewAuthenticator("sp", "/k.pem", "kid", "", "", "", nil)
+	err := a.Authenticate(&http.Request{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "kernel backend")
+}

connector.go

@@ -38,6 +38,19 @@ type interactiveU2MAuthenticator interface {
 	U2MClientID() string
 }
 
+// kernelOnlyAuthenticator is satisfied only by the JWT private-key M2M
+// authenticator, whose Authenticate always returns a kernel-only error (the
+// assertion is signed by the native kernel, not the Go path). Matches the
+// JWTM2MCredentials signature the kernel backend asserts structurally. Used to
+// skip telemetry on the kernel JWT M2M path: telemetry/feature-flag HTTP calls
+// go through cfg.Authenticator.Authenticate, which for this authenticator would
+// fail every request (burning a failing round-trip at connect). Telemetry is
+// best-effort, so it's dropped here rather than made to fail — consistent with
+// the U2M skip below and the Python/Node kernel paths (PECOBLR-3839).
+type kernelOnlyAuthenticator interface {
+	JWTM2MCredentials() (clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string)
+}
+
 // Connect returns a connection to the Databricks database from a connection pool.
 func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	defer debuglog.Track(ctx, "connector.Connect", "host=%s", c.cfg.Host)()
@@ -103,6 +116,12 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 		if _, isU2M := c.cfg.Authenticator.(interactiveU2MAuthenticator); isU2M {
 			skipTelemetry = true
 			log.Debug().Msg("telemetry skipped: kernel U2M owns the interactive auth flow")
+		} else if _, isKernelOnly := c.cfg.Authenticator.(kernelOnlyAuthenticator); isKernelOnly {
+			// JWT private-key M2M: Authenticate always errors on the Go path (the
+			// kernel signs the assertion), so every telemetry/feature-flag request
+			// would fail auth. Drop telemetry rather than burn failing round-trips.
+			skipTelemetry = true
+			log.Debug().Msg("telemetry skipped: kernel JWT M2M signs via the native kernel; Go-path auth is kernel-only")
 		}
 	}
 

connector_kernel_jwt.go

@@ -0,0 +1,73 @@
+//go:build cgo && databricks_kernel
+
+package dbsql
+
+import (
+	"github.com/databricks/databricks-sql-go/auth/oauth/jwtm2m"
+	"github.com/databricks/databricks-sql-go/internal/config"
+	"github.com/databricks/databricks-sql-go/logger"
+)
+
+// JWTPrivateKeyM2MConfig configures OAuth machine-to-machine authentication via
+// a JWT private-key client assertion (RFC 7523). A struct is used (rather than
+// positional args) so the many string fields can't be transposed.
+//
+// KERNEL BACKEND ONLY: the assertion is signed by the native kernel, so both
+// this type and WithJWTPrivateKeyM2M exist only in a build compiled with the
+// kernel backend (`cgo && databricks_kernel`). A default (Thrift-only) build
+// does not expose them, so a Thrift user can't configure an auth mode the
+// Thrift path cannot honor.
+type JWTPrivateKeyM2MConfig struct {
+	// ClientID is the service principal / OAuth client id (the assertion
+	// issuer and subject).
+	ClientID string
+	// KeyFile is the path to the PEM-encoded private key that signs the
+	// assertion.
+	KeyFile string
+	// Kid is the key id written into the JWT header so the IdP can select the
+	// registered public key.
+	Kid string
+	// Passphrase decrypts an encrypted PKCS#8 key; leave empty for an
+	// unencrypted key.
+	Passphrase string
+	// Algorithm is the JWT signing algorithm (RS256/384/512, PS256/384/512,
+	// ES256, ES384); empty defaults to RS256.
+	Algorithm string
+	// TokenURL is the OAuth IdP token endpoint. Required when the workspace's
+	// OAuth authority is an external IdP (e.g. Entra ID for Azure Databricks),
+	// since Databricks-native OIDC does not advertise the private_key_jwt
+	// method; empty falls back to the kernel's OIDC discovery.
+	TokenURL string
+	// Scopes overrides the requested OAuth scopes; empty uses the kernel
+	// default (all-apis).
+	Scopes []string
+}
+
+// WithJWTPrivateKeyM2M sets up OAuth M2M authentication using a JWT private-key
+// client assertion. See JWTPrivateKeyM2MConfig. Requires the kernel backend
+// (WithUseKernel(true)); the kernel signs the assertion. This option is only
+// compiled into a kernel-enabled build (`cgo && databricks_kernel`) — the
+// pure-Go Thrift path has no JWT-signing implementation, so exposing it there
+// would only add a divergent, non-functional surface.
+func WithJWTPrivateKeyM2M(cfg JWTPrivateKeyM2MConfig) ConnOption {
+	return func(c *config.Config) {
+		if cfg.ClientID != "" && cfg.KeyFile != "" && cfg.Kid != "" {
+			c.Authenticator = jwtm2m.NewAuthenticator(
+				cfg.ClientID, cfg.KeyFile, cfg.Kid, cfg.Passphrase, cfg.Algorithm, cfg.TokenURL, cfg.Scopes,
+			)
+			return
+		}
+		// Consistent with WithClientCredentials/WithAccessToken, a fully-empty
+		// config is a no-op (the caller simply didn't pick this option). But a
+		// PARTIAL config (some required fields set, others blank — e.g. a typo or
+		// an unset env var) is almost certainly a mistake: no authenticator gets
+		// installed, so connect later fails with a misleading "requires a personal
+		// access token" (or authenticates as PAT if WithAccessToken was also set).
+		// Warn so the misconfiguration isn't wholly invisible, rather than
+		// silently dropping it.
+		if cfg.ClientID != "" || cfg.KeyFile != "" || cfg.Kid != "" {
+			logger.Warn().Msg("WithJWTPrivateKeyM2M: incomplete config; ClientID, KeyFile, and Kid are all " +
+				"required. No JWT authenticator was installed — the connection will not use JWT private-key M2M.")
+		}
+	}
+}

connector_kernel_jwt_test.go

@@ -0,0 +1,46 @@
+//go:build cgo && databricks_kernel
+
+package dbsql
+
+import (
+	"testing"
+
+	"github.com/databricks/databricks-sql-go/internal/config"
+)
+
+func TestWithJWTPrivateKeyM2M(t *testing.T) {
+	apply := func(cfg JWTPrivateKeyM2MConfig) *config.Config {
+		c := &config.Config{}
+		WithJWTPrivateKeyM2M(cfg)(c)
+		return c
+	}
+
+	t.Run("full config installs the JWT authenticator", func(t *testing.T) {
+		c := apply(JWTPrivateKeyM2MConfig{ClientID: "sp", KeyFile: "/k.pem", Kid: "kid-1"})
+		if c.Authenticator == nil {
+			t.Fatal("expected an authenticator to be installed for a complete JWT config")
+		}
+		// It must satisfy the interface the kernel backend asserts to select JWT M2M.
+		if _, ok := c.Authenticator.(interface {
+			JWTM2MCredentials() (string, string, string, string, string, string, []string)
+		}); !ok {
+			t.Fatalf("installed authenticator %T does not expose JWTM2MCredentials", c.Authenticator)
+		}
+	})
+
+	t.Run("empty config is a silent no-op", func(t *testing.T) {
+		c := apply(JWTPrivateKeyM2MConfig{})
+		if c.Authenticator != nil {
+			t.Fatalf("expected no authenticator for an empty config, got %T", c.Authenticator)
+		}
+	})
+
+	t.Run("partial config installs no authenticator", func(t *testing.T) {
+		// ClientID + KeyFile but no Kid: the required trio is incomplete, so no
+		// authenticator is installed (and WithJWTPrivateKeyM2M warns).
+		c := apply(JWTPrivateKeyM2MConfig{ClientID: "sp", KeyFile: "/k.pem"})
+		if c.Authenticator != nil {
+			t.Fatalf("expected no authenticator for a partial config, got %T", c.Authenticator)
+		}
+	})
+}

doc.go

@@ -92,6 +92,7 @@ Supported functional options include:
   - WithMaxDownloadThreads (<num_threads> int). Sets up the max number of concurrent workers for cloud fetch. Default is 10. Optional
   - WithAuthenticator (<authenticator> auth.Authenticator). Sets up authentication. Required if neither access token or client credentials are provided.
   - WithClientCredentials(<clientID> string, <clientSecret> string). Sets up Oauth M2M authentication.
+  - WithJWTPrivateKeyM2M(<config> JWTPrivateKeyM2MConfig). Sets up OAuth M2M authentication with a JWT private-key client assertion (RFC 7523) instead of a client secret. Kernel backend only (requires WithUseKernel(true) and a build with -tags databricks_kernel); the kernel signs the assertion. See the kernel-backend section. Optional
   - WithUseKernel(<useKernel> bool). Routes execution through the SEA-via-kernel backend instead of Thrift. Requires a build with -tags databricks_kernel (CGO_ENABLED=1); the default build returns a clear error. Default is false. See the kernel-backend section below. Optional
   - WithWarehouseID(<id> string). The bare SQL warehouse id used by the kernel backend in preference to the http path; ignored by the Thrift backend. Optional
 
@@ -209,8 +210,9 @@ defers to RUST_LOG. Filter on the target databricks::sql::kernel (note the colon
 	# kernel logs plus its HTTP stack:
 	DBSQL_KERNEL_DEBUG=1 RUST_LOG=debug ./your_app 2>&1
 
-Supported on the kernel backend: PAT and OAuth (M2M via WithClientCredentials, U2M
-via the authType=oauthU2M DSN param); reading scalar, nested, and complex-typed
+Supported on the kernel backend: PAT and OAuth (M2M via WithClientCredentials, JWT
+private-key M2M via WithJWTPrivateKeyM2M, U2M via the authType=oauthU2M DSN param);
+reading scalar, nested, and complex-typed
 results (CloudFetch is transparent); bound query parameters (positional and named);
 context cancellation during execute; the initial namespace (WithInitialNamespace,
 applied post-connect via USE CATALOG / USE SCHEMA); metric-view metadata

internal/backend/kernel/auth.go

@@ -10,9 +10,10 @@ package kernel
 type AuthMode int
 
 const (
-	AuthPAT AuthMode = iota // personal access token
-	AuthM2M                 // OAuth client-credentials (client id + secret)
-	AuthU2M                 // OAuth user-to-machine (browser/PKCE; kernel-owned flow)
+	AuthPAT    AuthMode = iota // personal access token
+	AuthM2M                    // OAuth client-credentials (client id + secret)
+	AuthU2M                    // OAuth user-to-machine (browser/PKCE; kernel-owned flow)
+	AuthJWTM2M                 // OAuth client-credentials via a JWT private-key client assertion
 )
 
 // Auth is the resolved auth descriptor for a kernel connection. Only the fields
@@ -29,10 +30,18 @@ const (
 type Auth struct {
 	Mode         AuthMode
 	Token        string   // PAT
-	ClientID     string   // M2M + U2M (U2M: the cloud-inferred Go client id)
+	ClientID     string   // M2M + U2M + JWT M2M (U2M: the cloud-inferred Go client id)
 	ClientSecret string   // M2M
-	Scopes       []string // U2M — Thrift-parity scopes from oauth.GetScopes; nil → kernel default
+	Scopes       []string // U2M / JWT M2M — nil → kernel default
 	RedirectPort uint16   // U2M — no user option today; 0 → kernel default port (8020)
+
+	// JWT private-key M2M (Mode == AuthJWTM2M). The kernel signs a short-lived
+	// client assertion with the private key instead of sending a secret.
+	JWTKeyFile    string // path to the PEM private key (required)
+	JWTKid        string // key id written into the JWT header (required)
+	JWTPassphrase string // passphrase for an encrypted PKCS#8 key ("" → unencrypted)
+	JWTAlgorithm  string // signing algorithm ("" → kernel default RS256)
+	TokenURL      string // OAuth IdP token endpoint ("" → kernel OIDC discovery)
 }
 
 // M2MCredentialsProvider is implemented by the OAuth M2M authenticator to expose
@@ -78,3 +87,18 @@ type U2MCredentialsProvider interface {
 	// U2MClientID returns the OAuth client id for the U2M browser flow.
 	U2MClientID() string
 }
+
+// JWTM2MCredentialsProvider is implemented by the JWT private-key M2M
+// authenticator to expose the assertion-signing inputs. The kernel backend reads
+// these to drive the kernel's own JWT client-assertion flow (the kernel signs the
+// assertion and owns the token exchange), rather than using the authenticator's
+// Authenticate method — which is unsupported on the pure-Go Thrift path anyway.
+// Internal for the same reason as M2MCredentialsProvider (the key-reading
+// capability is not part of the driver's public API); satisfied structurally by
+// the unexported jwtm2m authenticator.
+type JWTM2MCredentialsProvider interface {
+	// JWTM2MCredentials returns the client id, private-key file path, key id,
+	// passphrase (may be ""), algorithm (may be ""), token URL (may be ""), and
+	// scopes (may be nil).
+	JWTM2MCredentials() (clientID, keyFile, kid, passphrase, algorithm, tokenURL string, scopes []string)
+}

internal/backend/kernel/backend.go

@@ -411,6 +411,33 @@ func (k *KernelBackend) setAuth(cfg *C.KernelSessionConfig) error {
 		}); err != nil {
 			return fmt.Errorf("kernel: set_auth_u2m: %w", toConnError(err))
 		}
+	case AuthJWTM2M:
+		// JWT private-key client assertion (RFC 7523). client_id / jwt_key_file /
+		// jwt_kid are required; passphrase / algorithm / scopes / token_url are
+		// optional — NULL lets the kernel fill its defaults (unencrypted key, RS256,
+		// all-apis, OIDC discovery). token_url points the grant at the workspace's
+		// OAuth IdP (e.g. Entra ID) when Databricks-native OIDC can't serve it.
+		clientID := newCStr(k.cfg.Auth.ClientID)
+		defer clientID.free()
+		keyFile := newCStr(k.cfg.Auth.JWTKeyFile)
+		defer keyFile.free()
+		kid := newCStr(k.cfg.Auth.JWTKid)
+		defer kid.free()
+		passphrase := newCStrOrNull(k.cfg.Auth.JWTPassphrase)
+		defer passphrase.free()
+		algorithm := newCStrOrNull(k.cfg.Auth.JWTAlgorithm)
+		defer algorithm.free()
+		scopes := newCStrOrNull(joinScopes(k.cfg.Auth.Scopes))
+		defer scopes.free()
+		tokenURL := newCStrOrNull(k.cfg.Auth.TokenURL)
+		defer tokenURL.free()
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_auth_m2m_jwt(
+				cfg, clientID.c, keyFile.c, kid.c, passphrase.c, algorithm.c, scopes.c, tokenURL.c,
+			)
+		}); err != nil {
+			return fmt.Errorf("kernel: set_auth_m2m_jwt: %w", toConnError(err))
+		}
 	default: // AuthPAT
 		tok := newCStr(k.cfg.Auth.Token)
 		defer tok.free()