chore(sqla2): bump SQLAlchemy-dependent drivers to dual-compat versions

docs/scripts/generate-database-docs.mjs

@@ -101,21 +101,29 @@ import json
 import ast
 import os
 
-def eval_node(node):
-    """Safely evaluate an AST node as a Python literal."""
+def eval_node(node, constants=None):
+    """Safely evaluate an AST node as a Python literal.
+
+    \`constants\` is an optional dict of module-level constant names -> already
+    -resolved Python values. It lets us resolve references like
+    \`AURORA_DATA_API_KNOWN_INCOMPATIBILITIES\` that point at a list/dict
+    defined (and potentially imported across files) elsewhere in
+    db_engine_specs, instead of falling through to the bare identifier
+    string.
+    """
     if node is None:
         return None
     if isinstance(node, ast.Constant):
         return node.value
     elif isinstance(node, ast.List):
-        return [eval_node(e) for e in node.elts]
+        return [eval_node(e, constants) for e in node.elts]
     elif isinstance(node, ast.Dict):
         result = {}
         for k, v in zip(node.keys, node.values):
             if k is not None:
-                key = eval_node(k)
+                key = eval_node(k, constants)
                 if key is not None:
-                    result[key] = eval_node(v)
+                    result[key] = eval_node(v, constants)
         return result
     elif isinstance(node, ast.Name):
         # Handle True, False, None constants
@@ -125,12 +133,14 @@ def eval_node(node):
             return False
         elif node.id == 'None':
             return None
+        if constants and node.id in constants:
+            return constants[node.id]
         return node.id
     elif isinstance(node, ast.Attribute):
         # Handle DatabaseCategory.SOMETHING - return just the attribute name
         return node.attr
     elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
-        left, right = eval_node(node.left), eval_node(node.right)
+        left, right = eval_node(node.left, constants), eval_node(node.right, constants)
         if isinstance(left, str) and isinstance(right, str):
             return left + right
         return None
@@ -274,6 +284,37 @@ CAP_METHODS = {
 # Intermediate base classes (e.g. PrestoBaseEngineSpec) do count as overrides.
 TRUE_BASE_CLASS = 'BaseEngineSpec'
 
+# Pass 0: collect module-level literal constants across every engine spec
+# file (e.g. AURORA_DATA_API_KNOWN_INCOMPATIBILITIES in base.py, imported
+# into mysql.py's \`compatible_databases\` metadata) so \`metadata\` dicts
+# that reference a shared constant by name resolve to its actual value
+# instead of the bare identifier string. Only module-scope assignments
+# (tree.body, not nested in classes/functions) are considered.
+MODULE_CONSTANTS = {}
+for filename in sorted(os.listdir(specs_dir)):
+    if not filename.endswith('.py') or filename in ('__init__.py', 'lib.py', 'lint_metadata.py'):
+        continue
+    filepath = os.path.join(specs_dir, filename)
+    try:
+        with open(filepath) as f:
+            source = f.read()
+        tree = ast.parse(source)
+        for item in tree.body:
+            targets = []
+            if isinstance(item, ast.Assign):
+                targets = item.targets
+            elif isinstance(item, ast.AnnAssign) and item.value is not None:
+                # Handle annotated module-level constants, e.g.
+                # \`AURORA_DATA_API_KNOWN_INCOMPATIBILITIES: list[KnownIncompatibility] = [...]\`
+                targets = [item.target]
+            for target in targets:
+                if isinstance(target, ast.Name) and target.id.isupper():
+                    val = eval_node(item.value, MODULE_CONSTANTS)
+                    if val is not None:
+                        MODULE_CONSTANTS[target.id] = val
+    except Exception:
+        continue
+
 # First pass: collect all class info (name, bases, metadata, cap_attrs, direct_methods)
 class_info = {}  # class_name -> {bases: [], metadata: {}, engine_name: str, filename: str, ...}
 
@@ -330,7 +371,7 @@ for filename in sorted(os.listdir(specs_dir)):
                             if isinstance(val, str):
                                 engine_attr = val
                         elif target.id == 'metadata':
-                            metadata = eval_node(item.value)
+                            metadata = eval_node(item.value, MODULE_CONSTANTS)
                         elif target.id in CAP_ATTR_DEFAULTS:
                             val = eval_node(item.value)
                             if isinstance(val, bool):

docs/src/components/databases/DatabasePage.tsx

@@ -41,7 +41,7 @@ import {
   GithubOutlined,
   BugOutlined,
 } from '@ant-design/icons';
-import type { DatabaseInfo } from './types';
+import type { DatabaseInfo, KnownIncompatibility } from './types';
 
 // Simple code block component for connection strings
 const CodeBlock: React.FC<{ children: React.ReactNode }> = ({ children }) => (
@@ -253,6 +253,53 @@ const DatabasePage: React.FC<DatabasePageProps> = ({ database, name }) => {
     );
   };
 
+  // Render known incompatibilities with a Superset dependency (e.g. a driver
+  // that doesn't yet support SQLAlchemy 2.0). Shared between the top-level
+  // documentation and each compatible-database entry.
+  const renderKnownIncompatibilities = (
+    incompatibilities?: KnownIncompatibility[],
+  ) => {
+    if (!incompatibilities?.length) return null;
+
+    return (
+      <Space direction="vertical" style={{ width: '100%' }}>
+        {incompatibilities.map((incompat, idx) => (
+          <Alert
+            key={idx}
+            type="warning"
+            showIcon
+            message={incompat.dependency}
+            description={
+              <>
+                {incompat.reason && (
+                  <Paragraph style={{ marginBottom: 4 }}>
+                    {incompat.reason}
+                  </Paragraph>
+                )}
+                <Space size="middle">
+                  {incompat.tracking_url && (
+                    <a
+                      href={incompat.tracking_url}
+                      target="_blank"
+                      rel="noreferrer"
+                    >
+                      <LinkOutlined /> Tracking issue
+                    </a>
+                  )}
+                  {incompat.since && (
+                    <Text type="secondary">
+                      Last confirmed: {incompat.since}
+                    </Text>
+                  )}
+                </Space>
+              </>
+            }
+          />
+        ))}
+      </Space>
+    );
+  };
+
   // Render compatible databases (for PostgreSQL, etc.)
   const renderCompatibleDatabases = () => {
     if (!docs?.compatible_databases?.length) return null;
@@ -320,6 +367,16 @@ const DatabasePage: React.FC<DatabasePageProps> = ({ database, name }) => {
                     />
                   </div>
                 )}
+                {compat.known_incompatibilities?.length > 0 && (
+                  <div style={{ marginTop: 16 }}>
+                    <Text strong>Known Incompatibilities:</Text>
+                    <div style={{ marginTop: 8 }}>
+                      {renderKnownIncompatibilities(
+                        compat.known_incompatibilities,
+                      )}
+                    </div>
+                  </div>
+                )}
                 {compat.notes && (
                   <Alert
                     message={compat.notes}
@@ -624,6 +681,17 @@ const DatabasePage: React.FC<DatabasePageProps> = ({ database, name }) => {
         </Card>
       )}
 
+      {/* Known Incompatibilities */}
+      {docs?.known_incompatibilities?.length > 0 && (
+        <Card
+          title="Known Incompatibilities"
+          style={{ marginBottom: 16 }}
+          type="inner"
+        >
+          {renderKnownIncompatibilities(docs.known_incompatibilities)}
+        </Card>
+      )}
+
       {/* Installation */}
       {(docs?.pypi_packages?.length || docs?.install_instructions) && (
         <Card title="Installation" style={{ marginBottom: 16 }}>

docs/src/components/databases/types.ts

@@ -72,6 +72,13 @@ export interface SSLConfiguration {
   };
 }
 
+export interface KnownIncompatibility {
+  dependency: string;  // e.g. "SQLAlchemy 2.0"
+  reason?: string;
+  tracking_url?: string;  // upstream issue/PR tracking a fix, if one exists
+  since?: string;  // ISO date this was last confirmed still broken
+}
+
 export interface CompatibleDatabase {
   name: string;
   description?: string;
@@ -84,6 +91,7 @@ export interface CompatibleDatabase {
   connection_examples?: ConnectionExample[];
   notes?: string;
   docs_url?: string;
+  known_incompatibilities?: KnownIncompatibility[];
 }
 
 export interface CustomError {
@@ -123,6 +131,7 @@ export interface DatabaseDocumentation {
   advanced_features?: Record<string, string>;
   compatible_databases?: CompatibleDatabase[];
   custom_errors?: CustomError[]; // Database-specific error messages and troubleshooting info
+  known_incompatibilities?: KnownIncompatibility[]; // Unresolved incompatibilities with a Superset dependency
 }
 
 export interface TimeGrains {

pyproject.toml

@@ -60,6 +60,15 @@ dependencies = [
     "flask-login>=0.6.0, < 1.0",
     "flask-migrate>=4.1.0, <5.0",
     "flask-session>=0.4.0, <1.0",
+    # Pinned explicitly below 3.0: 3.0.5 resolves without conflict and
+    # supports both SQLAlchemy 1.4 and 2.0, but real CI runs surfaced a
+    # structural incompatibility with Superset's current session/app-context
+    # handling across Celery task boundaries (see PR #42542) -- widespread
+    # "NoneType has no attribute X" failures and MySQL lock-wait timeouts,
+    # not just a connection-pool quirk. Needs dedicated investigation, not a
+    # driver-compat-prep bump; revisit alongside the actual SQLAlchemy 2.0
+    # core bump (discussion #40273, step 6).
+    "flask-sqlalchemy>=2.5.1, <3.0",
     "flask-wtf>=1.3.0, <2.0",
     "geopy",
     "greenlet<=3.5.4, >=3.5.4",
@@ -123,15 +132,25 @@ dependencies = [
 [project.optional-dependencies]
 
 athena = ["pyathena[pandas]>=3.35.2, <4"]
+# No SQLAlchemy 2.0 support anywhere in this dialect's ecosystem today: our
+# own preset-io/sqlalchemy-aurora-data-api fork is dead since 2021, and the
+# more active community fork (cloud-utils/sqlalchemy-aurora-data-api) has an
+# unresolved SQLAlchemy 2.0 break (upstream issue #43). See
+# superset/db_engine_specs/aurora.py's known_incompatibilities metadata.
 aurora-data-api = ["preset-sqlalchemy-aurora-data-api>=0.2.8,<0.3"]
 bigquery = [
     "pandas-gbq>=0.35.0",
-    "sqlalchemy-bigquery>=1.17.0",
+    # 1.17.1 is likely the final release: googleapis/python-bigquery-sqlalchemy
+    # was archived 2026-05-16. Both 1.17.0 and 1.17.1 support SQLAlchemy 1.4/2.0.
+    "sqlalchemy-bigquery>=1.17.1",
     "google-cloud-bigquery>=3.42.2",
 ]
 clickhouse = ["clickhouse-connect>=1.6.0, <2.0"]
 cockroachdb = ["cockroachdb>=0.3.5, <0.4"]
-crate = ["sqlalchemy-cratedb>=0.41.0, <1"]
+crate = ["sqlalchemy-cratedb>=0.43.1, <1"]
+# sqlalchemy-d1's only release (0.1.0, Nov 2025) pins sqlalchemy<2,>=1.4,
+# explicitly excluding SQLAlchemy 2.0. See superset/db_engine_specs/d1.py's
+# known_incompatibilities metadata.
 d1 = [
     "superset-engine-d1>=0.1.0",
     "sqlalchemy-d1>=0.1.0",
@@ -145,14 +164,27 @@ databricks = [
 datafusion = ["flightsql-dbapi>=0.2.2, <0.3"]
 db2 = ["ibm-db-sa<=0.4.4, >=0.4.4"]
 denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"]
-dremio = ["sqlalchemy-dremio>=1.2.1, <4"]
-drill = ["sqlalchemy-drill>=1.1.10, <2"]
+# sqlalchemy-dremio 3.0.5+ hard-pins sqlalchemy~=2.0.41, dropping 1.4; 3.0.4
+# is the last dual-compat release. Capped below 3.0.5 for now; widen back to
+# <4 in lockstep with Superset's own SQLAlchemy 2.0 core bump (discussion
+# #40273), not before.
+dremio = ["sqlalchemy-dremio>=1.2.1, <3.0.5"]
+# <2 was an artificial ceiling; upstream has no SQLAlchemy version cap and
+# 1.1.10 already supports SQLAlchemy 2.0 (added `import_dbapi` in 1.1.7).
+drill = ["sqlalchemy-drill>=1.1.10, <3"]
 druid = ["pydruid>=0.6.5,<0.7"]
 duckdb = ["duckdb>=1.5.4,<2", "duckdb-engine>=0.17.0"]
 dynamodb = ["pydynamodb>=0.8.2"]
-solr = ["sqlalchemy-solr >= 0.2.4.3"]
+# Effectively unmaintained (only dependabot bumps since 2024); hard-pinned to
+# SQLAlchemy ~1.4.7 upstream, no SQLAlchemy 2.0 work. See
+# superset/db_engine_specs/solr.py's known_incompatibilities metadata.
+solr = ["sqlalchemy-solr>=0.2.4.3"]
 elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"]
-exasol = ["sqlalchemy-exasol>=2.4.0, <8.0"]
+# sqlalchemy-exasol cuts hard from SQLAlchemy 1.4-only (<6.0.0) to 2.0-only
+# (>=6.0.0) with no dual-compat release. Capped below 6.0.0 for now; bump to
+# >=6.0.0,<8.0 in lockstep with Superset's own SQLAlchemy 2.0 core bump
+# (discussion #40273), not before.
+exasol = ["sqlalchemy-exasol>=2.4.0, <6.0.0"]
 excel = ["xlrd>=2.0.2, <2.1"]
 # Async dashboard "Export Data/Images to Excel": uploads the workbook to S3 and
 # emails a pre-signed link. boto3 is imported lazily by superset.utils.s3, so
@@ -165,8 +197,12 @@ fastmcp = [
     # heuristic that under-counts JSON-heavy MCP responses.
     "tiktoken>=0.13.0,<1.0",
 ]
-firebird = ["sqlalchemy-firebird>=0.8.0, <2.2"]
-firebolt = ["firebolt-sqlalchemy>=1.0.0, <2"]
+# sqlalchemy-firebird >=2.0.0 unconditionally requires SQLAlchemy 2.0 on
+# Python >=3.8 (which covers Superset's >=3.11 floor), with no dual-compat
+# release. Capped below 2.0.0 for now; bump to >=2.2.0 in lockstep with
+# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before.
+firebird = ["sqlalchemy-firebird>=0.8.0, <2.0.0"]
+firebolt = ["firebolt-sqlalchemy>=1.1.2, <2"]
 gevent = ["gevent>=26.4.0"]
 gsheets = ["shillelagh[gsheetsapi]>=1.4.4, <2"]
 hana = ["hdbcli==2.29.25", "sqlalchemy_hana==3.0.3"]
@@ -177,14 +213,20 @@ hive = [
     "thrift_sasl>=0.4.3, < 1.0.0",
 ]
 impala = ["impyla>=0.24.0, <0.25"]
+# Actively maintained upstream, but setup.py on main hard-pins
+# sqlalchemy==1.4.*, no SQLAlchemy 2.0 work yet. See
+# superset/db_engine_specs/kusto.py's known_incompatibilities metadata.
 kusto = ["sqlalchemy-kusto>=3.1.2, <4"]
 kylin = ["kylinpy>=2.8.4, <2.9"]
 mssql = ["pymssql>=2.3.13, <3"]
 # motherduck is an alias for duckdb - MotherDuck works via the duckdb driver
 motherduck = ["apache-superset[duckdb]"]
 mysql = ["mysqlclient>=2.2.8, <3"]
 ocient = [
-    "sqlalchemy-ocient>=1.0.0, <4",
+    # Closed-source vendor package with no public changelog; permissive
+    # unpinned sqlalchemy>=1.4 declared, but SQLAlchemy 2.0 support is
+    # unverified. Lower confidence than the other bumps in this PR.
+    "sqlalchemy-ocient>=3.0.0, <4",
     "pyocient>=1.0.15, <4",
     "shapely",
     "geojson",
@@ -197,8 +239,16 @@ postgres = ["psycopg2-binary==2.9.12"]
 presto = ["pyhive[presto]>=0.6.5"]
 trino = ["trino>=0.338.0"]
 prophet = ["prophet>=1.3.0, <2"]
+# sqlalchemy-redshift cuts hard from SQLAlchemy 1.4-only (0.8.x) to 2.0-only
+# (>=1.0.0) with no dual-compat release; the existing <0.9 ceiling already
+# keeps this on the 1.4-only line. Bump to >=1.0.0 in lockstep with
+# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before.
 redshift = ["sqlalchemy-redshift>=0.8.1, <0.9"]
-risingwave = ["sqlalchemy-risingwave"]
+# No release of sqlalchemy-risingwave has ever supported both SQLAlchemy 1.4
+# and 2.0 (version numbers don't track SQLAlchemy compat monotonically); pin
+# to the newest 1.4-only release for now. Bump to >=2.0.0 in lockstep with
+# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before.
+risingwave = ["sqlalchemy-risingwave>=1.4.1, <2.0.0"]
 shillelagh = ["shillelagh[all]>=1.4.4, <2"]
 singlestore = ["sqlalchemy-singlestoredb>=1.2.1, <2"]
 snowflake = ["snowflake-sqlalchemy>=1.11.0, <2"]

requirements/base.txt

@@ -150,6 +150,7 @@ flask-session==0.8.0
     # via apache-superset (pyproject.toml)
 flask-sqlalchemy==2.5.1
     # via
+    #   apache-superset (pyproject.toml)
     #   flask-appbuilder
     #   flask-migrate
 flask-talisman==1.1.0

requirements/development.txt

@@ -312,6 +312,7 @@ flask-session==0.8.0
 flask-sqlalchemy==2.5.1
     # via
     #   -c requirements/base-constraint.txt
+    #   apache-superset
     #   flask-appbuilder
     #   flask-migrate
 flask-talisman==1.1.0
@@ -989,7 +990,7 @@ sqlalchemy==1.4.54
     #   sqlalchemy-bigquery
     #   sqlalchemy-continuum
     #   sqlalchemy-utils
-sqlalchemy-bigquery==1.17.0
+sqlalchemy-bigquery==1.17.1
     # via apache-superset
 sqlalchemy-continuum==1.7.0
     # via