Support for external principals

CHANGELOG.md

@@ -28,6 +28,13 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 ## [Unreleased]
 
 ### Highlights
+- Polaris now fully supports "external" principals, that is, principals that are not backed by an 
+  entity in Polaris metastore. By enabling external principals, either globally or per-realm,
+  Polaris now skips the principal entity metastore lookup. This means that synchronizing principals 
+  between an external IDP and Polaris is not necessary anymore. To enable external principals, 
+  set the `polaris.authentication.credential-mode` option to `external`. Note: external principals
+  are not compatible with internal authentication and internal authorization; you must configure an
+  external IDP and an external PDP, such as OPA or Ranger.
 
 ### Upgrade notes
 

bom/build.gradle.kts

@@ -37,6 +37,7 @@ dependencies {
     api(project(":polaris-floci-az-testcontainer"))
     api(project(":polaris-floci-gcp-testcontainer"))
     api(project(":polaris-keycloak-testcontainer"))
+    api(project(":polaris-opa-testcontainer"))
     api(project(":polaris-rustfs-testcontainer"))
     api(project(":polaris-immutables"))
     api(project(":polaris-misc-types"))
@@ -121,6 +122,7 @@ dependencies {
     api(project(":polaris-runtime-defaults"))
     api(project(":polaris-server"))
     api(project(":polaris-runtime-service"))
+    api(project(":polaris-runtime-service-it"))
     api(project(":polaris-runtime-spark-tests"))
 
     api(project(":polaris-tests"))

extensions/auth/opa/build.gradle.kts

@@ -96,7 +96,7 @@ dependencies {
   opaStartupActionCompileOnly("org.apache.polaris.server-test-runner:polaris-server-test-runner")
   opaStartupActionImplementation(platform(libs.testcontainers.bom))
   opaStartupActionImplementation("org.testcontainers:testcontainers")
-  opaStartupActionImplementation(project(":polaris-container-spec-helper"))
+  opaStartupActionImplementation(project(":polaris-opa-testcontainer"))
 
   intTestBase(platform(libs.junit.bom))
   intTestBase("org.junit.jupiter:junit-jupiter")

extensions/auth/opa/src/opaStartupAction/java/org/apache/polaris/extension/auth/opa/test/OpaStartupAction.java

@@ -18,87 +18,18 @@
  */
 package org.apache.polaris.extension.auth.opa.test;
 
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.time.Duration;
-import org.apache.polaris.containerspec.ContainerSpecHelper;
 import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction;
 import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.wait.strategy.Wait;
+import org.apache.polaris.test.opa.OpaContainer;
 
 /** Starts an OPA test server before the external Polaris server process starts. */
 public class OpaStartupAction implements PolarisServerStartupAction {
-  private static final int OPA_PORT = 8181;
+
   private static final String POLICY_NAME = "polaris/authz";
   private static final String POLICY_PACKAGE = POLICY_NAME.replace('/', '.');
 
-  private GenericContainer<?> opa;
-
-  @Override
-  @SuppressWarnings("resource")
-  public void start(PolarisServerStartupContext context) {
-    opa =
-        new GenericContainer<>(
-                ContainerSpecHelper.containerSpecHelper("opa", OpaStartupAction.class)
-                    .dockerImageName(null))
-            .withExposedPorts(OPA_PORT)
-            .withCommand("run", "--server", "--addr=0.0.0.0:8181")
-            .waitingFor(
-                Wait.forHttp("/health")
-                    .forPort(OPA_PORT)
-                    .forStatusCode(200)
-                    .withStartupTimeout(Duration.ofSeconds(120)));
-
-    opa.start();
-
-    String baseUrl = "http://" + opa.getHost() + ":" + opa.getMappedPort(OPA_PORT);
-    loadRegoPolicy(baseUrl, POLICY_NAME, polarisRegoPolicy());
-    context
-        .getSystemProperties()
-        .put("polaris.authorization.opa.policy-uri", baseUrl + "/v1/data/" + POLICY_NAME);
-  }
-
-  @Override
-  public void close() {
-    if (opa != null) {
-      opa.stop();
-      opa = null;
-    }
-  }
-
-  private void loadRegoPolicy(String baseUrl, String policyName, String regoPolicy) {
-    try {
-      URL url = URI.create(baseUrl + "/v1/policies/" + policyName).toURL();
-      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-      conn.setRequestMethod("PUT");
-      conn.setDoOutput(true);
-      conn.setRequestProperty("Content-Type", "text/plain");
-
-      try (OutputStream os = conn.getOutputStream()) {
-        os.write(regoPolicy.getBytes(StandardCharsets.UTF_8));
-      }
-
-      int code = conn.getResponseCode();
-      if (code < 200 || code >= 300) {
-        throw new RuntimeException("OPA policy upload failed, HTTP " + code);
-      }
-    } catch (Exception e) {
-      String logs = "";
-      try {
-        logs = opa.getLogs();
-      } catch (Throwable ignored) {
-        // ignore logging failures while reporting the original startup failure
-      }
-      throw new RuntimeException("Failed to load OPA policy. Container logs:\n" + logs, e);
-    }
-  }
-
-  private String polarisRegoPolicy() {
-    return """
+  private static final String POLICY =
+      """
         package %s
 
         default allow := false
@@ -113,6 +44,27 @@ private String polarisRegoPolicy() {
           input.actor.principal == "admin"
         }
         """
-        .formatted(POLICY_PACKAGE);
+          .formatted(POLICY_PACKAGE);
+
+  private OpaContainer opa;
+
+  @Override
+  public void start(PolarisServerStartupContext context) {
+    opa = new OpaContainer();
+    opa.start();
+    opa.uploadRegoPolicy(POLICY_NAME, POLICY);
+    context
+        .getSystemProperties()
+        .put(
+            "polaris.authorization.opa.policy-uri",
+            opa.getExternalUrl() + "v1/data/" + POLICY_NAME);
+  }
+
+  @Override
+  public void close() {
+    if (opa != null) {
+      opa.stop();
+      opa = null;
+    }
   }
 }

gradle/projects.main.properties

@@ -29,6 +29,7 @@ polaris-runtime-defaults=runtime/defaults
 polaris-runtime-service=runtime/service
 polaris-server=runtime/server
 polaris-distribution=runtime/distribution
+polaris-runtime-service-it=runtime/service-it
 polaris-runtime-spark-tests=runtime/spark-tests
 polaris-admin=runtime/admin
 polaris-runtime-common=runtime/common
@@ -42,6 +43,7 @@ polaris-floci-aws-testcontainer=tools/testcontainers/floci-aws
 polaris-floci-az-testcontainer=tools/testcontainers/floci-az
 polaris-floci-gcp-testcontainer=tools/testcontainers/floci-gcp
 polaris-keycloak-testcontainer=tools/testcontainers/keycloak
+polaris-opa-testcontainer=tools/testcontainers/opa
 polaris-rustfs-testcontainer=tools/testcontainers/rustfs-testcontainer
 polaris-hms-testcontainer=tools/testcontainers/hms-testcontainer
 polaris-version=tools/version

helm/polaris/ci/authentication-values.yaml

@@ -67,3 +67,5 @@ oidc:
     secret:
       name: polaris-oidc
       key: client-secret
+  principalMapper:
+    nameClaimPath: preferred_username

polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisPrincipalAttributes.java

@@ -26,6 +26,22 @@ public final class PolarisPrincipalAttributes {
 
   private PolarisPrincipalAttributes() {}
 
+  /**
+   * Attribute key, of type {@link Boolean}, used to mark a principal as external.
+   *
+   * <p>When present and true, the principal is external: it has no backing entity in the Polaris
+   * metastore, and its roles are resolved directly from the authentication result rather than from
+   * metastore grants.
+   *
+   * <p>Authenticators must set this attribute to {@code true} when they deliberately take the
+   * external-principal path. The resolver treats an explicit {@code true} as external and a missing
+   * or false marker as internal.
+   *
+   * <p>Note: Callers must not assume that this attribute is always present.
+   */
+  public static final AttributeKey<Boolean> EXTERNAL_PRINCIPAL_ATTRIBUTE_KEY =
+      new AttributeKey<>("org.apache.polaris.core.auth.EXTERNAL_PRINCIPAL");
+
   /**
    * Attribute key for the principal entity attribute, of type {@link PrincipalEntity}.
    *

polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java

@@ -30,6 +30,8 @@
 import java.util.stream.Collectors;
 import org.apache.polaris.core.PolarisCallContext;
 import org.apache.polaris.core.PolarisDiagnostics;
+import org.apache.polaris.core.auth.AuthorizationRequest;
+import org.apache.polaris.core.auth.AuthorizationState;
 import org.apache.polaris.core.auth.PolarisPrincipal;
 import org.apache.polaris.core.auth.PolarisPrincipalAttributes;
 import org.apache.polaris.core.entity.CatalogEntity;
@@ -40,6 +42,8 @@
 import org.apache.polaris.core.entity.PolarisEntityType;
 import org.apache.polaris.core.entity.PolarisGrantRecord;
 import org.apache.polaris.core.entity.PolarisPrivilege;
+import org.apache.polaris.core.entity.PrincipalEntity;
+import org.apache.polaris.core.entity.PrincipalRoleEntity;
 import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
 import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
 import org.apache.polaris.core.persistence.cache.EntityCache;
@@ -56,6 +60,11 @@
  */
 public class Resolver {
 
+  // Sentinel ids for synthetic entities created for external principals. Negative so they never
+  // collide with real, positive entity ids. Role ids are derived by decrementing from the base.
+  private static final long EXTERNAL_PRINCIPAL_ID = -1L;
+  private static final long EXTERNAL_PRINCIPAL_ROLE_ID_BASE = -2L;
+
   // we stash the Polaris call context here
   private final @NonNull PolarisCallContext polarisCallContext;
 
@@ -781,6 +790,17 @@ private ResolverStatus resolvePaths(
   private ResolverStatus resolveCallerPrincipalAndPrincipalRoles(
       List<ResolvedPolarisEntity> toValidate, boolean resolvePrincipalRoles) {
 
+    // External principals are not backed by the metastore: synthesize the caller principal and its
+    // roles directly from the authenticated principal instead of resolving them from the backend.
+    boolean externalPrincipal =
+        polarisPrincipal
+            .getAttributes()
+            .getOptional(PolarisPrincipalAttributes.EXTERNAL_PRINCIPAL_ATTRIBUTE_KEY)
+            .orElse(false);
+    if (externalPrincipal) {
+      return resolveExternalCallerPrincipalAndPrincipalRoles(resolvePrincipalRoles);
+    }
+
     // resolve the principal, by name or id
     this.resolvedCallerPrincipal =
         this.resolveByName(toValidate, PolarisEntityType.PRINCIPAL, polarisPrincipal.getName());
@@ -811,6 +831,48 @@ private ResolverStatus resolveCallerPrincipalAndPrincipalRoles(
     return new ResolverStatus(ResolverStatus.StatusEnum.SUCCESS);
   }
 
+  /**
+   * Synthesize the caller principal and its principal roles for an external principal. The
+   * principal is assumed to exist and to be valid; neither it nor its roles are read from the
+   * metastore. Synthetic entities carry negative sentinel ids (so they never collide with real,
+   * positive entity ids) and empty grant records.
+   *
+   * <p>TODO: synthetic principal and roles are a temporary workaround for external principals,
+   * until external authorizers change their implementation of {@link
+   * org.apache.polaris.core.auth.PolarisAuthorizer#resolveAuthorizationInputs(AuthorizationState,
+   * AuthorizationRequest)} to avoid calling {@link PolarisResolutionManifest#resolveAll()}, and
+   * instead only resolve the securables that are actually needed for authorization, cf. {@link
+   * PolarisResolutionManifest#resolveSelections(Set)}.
+   */
+  private ResolverStatus resolveExternalCallerPrincipalAndPrincipalRoles(
+      boolean resolvePrincipalRoles) {
+    PrincipalEntity syntheticPrincipal =
+        new PrincipalEntity.Builder()
+            .setId(EXTERNAL_PRINCIPAL_ID)
+            .setName(polarisPrincipal.getName())
+            .build();
+    this.resolvedCallerPrincipal =
+        new ResolvedPolarisEntity(syntheticPrincipal, List.of(), List.of());
+    // Register by ID only: synthetic entities must not be indexed by name, or they would shadow
+    // real stored entities with the same name in subsequent resolveByName() lookups.
+    this.resolvedEntriesById.put(EXTERNAL_PRINCIPAL_ID, this.resolvedCallerPrincipal);
+
+    this.resolvedCallerPrincipalRoles = new ArrayList<>();
+    if (resolvePrincipalRoles) {
+      long roleId = EXTERNAL_PRINCIPAL_ROLE_ID_BASE;
+      for (String roleName : polarisPrincipal.getRoles()) {
+        PrincipalRoleEntity syntheticRole =
+            new PrincipalRoleEntity.Builder().setId(roleId).setName(roleName).build();
+        ResolvedPolarisEntity resolvedRole =
+            new ResolvedPolarisEntity(syntheticRole, List.of(), List.of());
+        this.resolvedEntriesById.put(roleId--, resolvedRole);
+        this.resolvedCallerPrincipalRoles.add(resolvedRole);
+      }
+    }
+
+    return new ResolverStatus(ResolverStatus.StatusEnum.SUCCESS);
+  }
+
   /**
    * Resolve all principal roles that the principal has grants for
    *