diff --git a/docs/scripts/generate-database-docs.mjs b/docs/scripts/generate-database-docs.mjs index 25c6e0784579..e43b4d487e8e 100644 --- a/docs/scripts/generate-database-docs.mjs +++ b/docs/scripts/generate-database-docs.mjs @@ -101,21 +101,29 @@ import json import ast import os -def eval_node(node): - """Safely evaluate an AST node as a Python literal.""" +def eval_node(node, constants=None): + """Safely evaluate an AST node as a Python literal. + + \`constants\` is an optional dict of module-level constant names -> already + -resolved Python values. It lets us resolve references like + \`AURORA_DATA_API_KNOWN_INCOMPATIBILITIES\` that point at a list/dict + defined (and potentially imported across files) elsewhere in + db_engine_specs, instead of falling through to the bare identifier + string. + """ if node is None: return None if isinstance(node, ast.Constant): return node.value elif isinstance(node, ast.List): - return [eval_node(e) for e in node.elts] + return [eval_node(e, constants) for e in node.elts] elif isinstance(node, ast.Dict): result = {} for k, v in zip(node.keys, node.values): if k is not None: - key = eval_node(k) + key = eval_node(k, constants) if key is not None: - result[key] = eval_node(v) + result[key] = eval_node(v, constants) return result elif isinstance(node, ast.Name): # Handle True, False, None constants @@ -125,12 +133,14 @@ def eval_node(node): return False elif node.id == 'None': return None + if constants and node.id in constants: + return constants[node.id] return node.id elif isinstance(node, ast.Attribute): # Handle DatabaseCategory.SOMETHING - return just the attribute name return node.attr elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): - left, right = eval_node(node.left), eval_node(node.right) + left, right = eval_node(node.left, constants), eval_node(node.right, constants) if isinstance(left, str) and isinstance(right, str): return left + right return None @@ -274,6 +284,37 @@ CAP_METHODS = { # Intermediate base classes (e.g. PrestoBaseEngineSpec) do count as overrides. TRUE_BASE_CLASS = 'BaseEngineSpec' +# Pass 0: collect module-level literal constants across every engine spec +# file (e.g. AURORA_DATA_API_KNOWN_INCOMPATIBILITIES in base.py, imported +# into mysql.py's \`compatible_databases\` metadata) so \`metadata\` dicts +# that reference a shared constant by name resolve to its actual value +# instead of the bare identifier string. Only module-scope assignments +# (tree.body, not nested in classes/functions) are considered. +MODULE_CONSTANTS = {} +for filename in sorted(os.listdir(specs_dir)): + if not filename.endswith('.py') or filename in ('__init__.py', 'lib.py', 'lint_metadata.py'): + continue + filepath = os.path.join(specs_dir, filename) + try: + with open(filepath) as f: + source = f.read() + tree = ast.parse(source) + for item in tree.body: + targets = [] + if isinstance(item, ast.Assign): + targets = item.targets + elif isinstance(item, ast.AnnAssign) and item.value is not None: + # Handle annotated module-level constants, e.g. + # \`AURORA_DATA_API_KNOWN_INCOMPATIBILITIES: list[KnownIncompatibility] = [...]\` + targets = [item.target] + for target in targets: + if isinstance(target, ast.Name) and target.id.isupper(): + val = eval_node(item.value, MODULE_CONSTANTS) + if val is not None: + MODULE_CONSTANTS[target.id] = val + except Exception: + continue + # First pass: collect all class info (name, bases, metadata, cap_attrs, direct_methods) class_info = {} # class_name -> {bases: [], metadata: {}, engine_name: str, filename: str, ...} @@ -330,7 +371,7 @@ for filename in sorted(os.listdir(specs_dir)): if isinstance(val, str): engine_attr = val elif target.id == 'metadata': - metadata = eval_node(item.value) + metadata = eval_node(item.value, MODULE_CONSTANTS) elif target.id in CAP_ATTR_DEFAULTS: val = eval_node(item.value) if isinstance(val, bool): diff --git a/docs/src/components/databases/DatabasePage.tsx b/docs/src/components/databases/DatabasePage.tsx index 8e4b8e836680..a1f293e11d62 100644 --- a/docs/src/components/databases/DatabasePage.tsx +++ b/docs/src/components/databases/DatabasePage.tsx @@ -41,7 +41,7 @@ import { GithubOutlined, BugOutlined, } from '@ant-design/icons'; -import type { DatabaseInfo } from './types'; +import type { DatabaseInfo, KnownIncompatibility } from './types'; // Simple code block component for connection strings const CodeBlock: React.FC<{ children: React.ReactNode }> = ({ children }) => ( @@ -253,6 +253,53 @@ const DatabasePage: React.FC = ({ database, name }) => { ); }; + // Render known incompatibilities with a Superset dependency (e.g. a driver + // that doesn't yet support SQLAlchemy 2.0). Shared between the top-level + // documentation and each compatible-database entry. + const renderKnownIncompatibilities = ( + incompatibilities?: KnownIncompatibility[], + ) => { + if (!incompatibilities?.length) return null; + + return ( + + {incompatibilities.map((incompat, idx) => ( + + {incompat.reason && ( + + {incompat.reason} + + )} + + {incompat.tracking_url && ( + + Tracking issue + + )} + {incompat.since && ( + + Last confirmed: {incompat.since} + + )} + + + } + /> + ))} + + ); + }; + // Render compatible databases (for PostgreSQL, etc.) const renderCompatibleDatabases = () => { if (!docs?.compatible_databases?.length) return null; @@ -320,6 +367,16 @@ const DatabasePage: React.FC = ({ database, name }) => { /> )} + {compat.known_incompatibilities?.length > 0 && ( +
+ Known Incompatibilities: +
+ {renderKnownIncompatibilities( + compat.known_incompatibilities, + )} +
+
+ )} {compat.notes && ( = ({ database, name }) => { )} + {/* Known Incompatibilities */} + {docs?.known_incompatibilities?.length > 0 && ( + + {renderKnownIncompatibilities(docs.known_incompatibilities)} + + )} + {/* Installation */} {(docs?.pypi_packages?.length || docs?.install_instructions) && ( diff --git a/docs/src/components/databases/types.ts b/docs/src/components/databases/types.ts index 7b3f61a57c5f..fb3689e9bc1e 100644 --- a/docs/src/components/databases/types.ts +++ b/docs/src/components/databases/types.ts @@ -72,6 +72,13 @@ export interface SSLConfiguration { }; } +export interface KnownIncompatibility { + dependency: string; // e.g. "SQLAlchemy 2.0" + reason?: string; + tracking_url?: string; // upstream issue/PR tracking a fix, if one exists + since?: string; // ISO date this was last confirmed still broken +} + export interface CompatibleDatabase { name: string; description?: string; @@ -84,6 +91,7 @@ export interface CompatibleDatabase { connection_examples?: ConnectionExample[]; notes?: string; docs_url?: string; + known_incompatibilities?: KnownIncompatibility[]; } export interface CustomError { @@ -123,6 +131,7 @@ export interface DatabaseDocumentation { advanced_features?: Record; compatible_databases?: CompatibleDatabase[]; custom_errors?: CustomError[]; // Database-specific error messages and troubleshooting info + known_incompatibilities?: KnownIncompatibility[]; // Unresolved incompatibilities with a Superset dependency } export interface TimeGrains { diff --git a/pyproject.toml b/pyproject.toml index 118a1fe2dcc7..6ea0496b2fc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,15 @@ dependencies = [ "flask-login>=0.6.0, < 1.0", "flask-migrate>=4.1.0, <5.0", "flask-session>=0.4.0, <1.0", + # Pinned explicitly below 3.0: 3.0.5 resolves without conflict and + # supports both SQLAlchemy 1.4 and 2.0, but real CI runs surfaced a + # structural incompatibility with Superset's current session/app-context + # handling across Celery task boundaries (see PR #42542) -- widespread + # "NoneType has no attribute X" failures and MySQL lock-wait timeouts, + # not just a connection-pool quirk. Needs dedicated investigation, not a + # driver-compat-prep bump; revisit alongside the actual SQLAlchemy 2.0 + # core bump (discussion #40273, step 6). + "flask-sqlalchemy>=2.5.1, <3.0", "flask-wtf>=1.3.0, <2.0", "geopy", "greenlet<=3.5.4, >=3.5.4", @@ -123,15 +132,25 @@ dependencies = [ [project.optional-dependencies] athena = ["pyathena[pandas]>=3.35.2, <4"] +# No SQLAlchemy 2.0 support anywhere in this dialect's ecosystem today: our +# own preset-io/sqlalchemy-aurora-data-api fork is dead since 2021, and the +# more active community fork (cloud-utils/sqlalchemy-aurora-data-api) has an +# unresolved SQLAlchemy 2.0 break (upstream issue #43). See +# superset/db_engine_specs/aurora.py's known_incompatibilities metadata. aurora-data-api = ["preset-sqlalchemy-aurora-data-api>=0.2.8,<0.3"] bigquery = [ "pandas-gbq>=0.35.0", - "sqlalchemy-bigquery>=1.17.0", + # 1.17.1 is likely the final release: googleapis/python-bigquery-sqlalchemy + # was archived 2026-05-16. Both 1.17.0 and 1.17.1 support SQLAlchemy 1.4/2.0. + "sqlalchemy-bigquery>=1.17.1", "google-cloud-bigquery>=3.42.2", ] clickhouse = ["clickhouse-connect>=1.6.0, <2.0"] cockroachdb = ["cockroachdb>=0.3.5, <0.4"] -crate = ["sqlalchemy-cratedb>=0.41.0, <1"] +crate = ["sqlalchemy-cratedb>=0.43.1, <1"] +# sqlalchemy-d1's only release (0.1.0, Nov 2025) pins sqlalchemy<2,>=1.4, +# explicitly excluding SQLAlchemy 2.0. See superset/db_engine_specs/d1.py's +# known_incompatibilities metadata. d1 = [ "superset-engine-d1>=0.1.0", "sqlalchemy-d1>=0.1.0", @@ -145,14 +164,27 @@ databricks = [ datafusion = ["flightsql-dbapi>=0.2.2, <0.3"] db2 = ["ibm-db-sa<=0.4.4, >=0.4.4"] denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"] -dremio = ["sqlalchemy-dremio>=1.2.1, <4"] -drill = ["sqlalchemy-drill>=1.1.10, <2"] +# sqlalchemy-dremio 3.0.5+ hard-pins sqlalchemy~=2.0.41, dropping 1.4; 3.0.4 +# is the last dual-compat release. Capped below 3.0.5 for now; widen back to +# <4 in lockstep with Superset's own SQLAlchemy 2.0 core bump (discussion +# #40273), not before. +dremio = ["sqlalchemy-dremio>=1.2.1, <3.0.5"] +# <2 was an artificial ceiling; upstream has no SQLAlchemy version cap and +# 1.1.10 already supports SQLAlchemy 2.0 (added `import_dbapi` in 1.1.7). +drill = ["sqlalchemy-drill>=1.1.10, <3"] druid = ["pydruid>=0.6.5,<0.7"] duckdb = ["duckdb>=1.5.4,<2", "duckdb-engine>=0.17.0"] dynamodb = ["pydynamodb>=0.8.2"] -solr = ["sqlalchemy-solr >= 0.2.4.3"] +# Effectively unmaintained (only dependabot bumps since 2024); hard-pinned to +# SQLAlchemy ~1.4.7 upstream, no SQLAlchemy 2.0 work. See +# superset/db_engine_specs/solr.py's known_incompatibilities metadata. +solr = ["sqlalchemy-solr>=0.2.4.3"] elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"] -exasol = ["sqlalchemy-exasol>=2.4.0, <8.0"] +# sqlalchemy-exasol cuts hard from SQLAlchemy 1.4-only (<6.0.0) to 2.0-only +# (>=6.0.0) with no dual-compat release. Capped below 6.0.0 for now; bump to +# >=6.0.0,<8.0 in lockstep with Superset's own SQLAlchemy 2.0 core bump +# (discussion #40273), not before. +exasol = ["sqlalchemy-exasol>=2.4.0, <6.0.0"] excel = ["xlrd>=2.0.2, <2.1"] # Async dashboard "Export Data/Images to Excel": uploads the workbook to S3 and # emails a pre-signed link. boto3 is imported lazily by superset.utils.s3, so @@ -165,8 +197,12 @@ fastmcp = [ # heuristic that under-counts JSON-heavy MCP responses. "tiktoken>=0.13.0,<1.0", ] -firebird = ["sqlalchemy-firebird>=0.8.0, <2.2"] -firebolt = ["firebolt-sqlalchemy>=1.0.0, <2"] +# sqlalchemy-firebird >=2.0.0 unconditionally requires SQLAlchemy 2.0 on +# Python >=3.8 (which covers Superset's >=3.11 floor), with no dual-compat +# release. Capped below 2.0.0 for now; bump to >=2.2.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. +firebird = ["sqlalchemy-firebird>=0.8.0, <2.0.0"] +firebolt = ["firebolt-sqlalchemy>=1.1.2, <2"] gevent = ["gevent>=26.4.0"] gsheets = ["shillelagh[gsheetsapi]>=1.4.4, <2"] hana = ["hdbcli==2.29.25", "sqlalchemy_hana==3.0.3"] @@ -177,6 +213,9 @@ hive = [ "thrift_sasl>=0.4.3, < 1.0.0", ] impala = ["impyla>=0.24.0, <0.25"] +# Actively maintained upstream, but setup.py on main hard-pins +# sqlalchemy==1.4.*, no SQLAlchemy 2.0 work yet. See +# superset/db_engine_specs/kusto.py's known_incompatibilities metadata. kusto = ["sqlalchemy-kusto>=3.1.2, <4"] kylin = ["kylinpy>=2.8.4, <2.9"] mssql = ["pymssql>=2.3.13, <3"] @@ -184,7 +223,10 @@ mssql = ["pymssql>=2.3.13, <3"] motherduck = ["apache-superset[duckdb]"] mysql = ["mysqlclient>=2.2.8, <3"] ocient = [ - "sqlalchemy-ocient>=1.0.0, <4", + # Closed-source vendor package with no public changelog; permissive + # unpinned sqlalchemy>=1.4 declared, but SQLAlchemy 2.0 support is + # unverified. Lower confidence than the other bumps in this PR. + "sqlalchemy-ocient>=3.0.0, <4", "pyocient>=1.0.15, <4", "shapely", "geojson", @@ -197,8 +239,16 @@ postgres = ["psycopg2-binary==2.9.12"] presto = ["pyhive[presto]>=0.6.5"] trino = ["trino>=0.338.0"] prophet = ["prophet>=1.3.0, <2"] +# sqlalchemy-redshift cuts hard from SQLAlchemy 1.4-only (0.8.x) to 2.0-only +# (>=1.0.0) with no dual-compat release; the existing <0.9 ceiling already +# keeps this on the 1.4-only line. Bump to >=1.0.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. redshift = ["sqlalchemy-redshift>=0.8.1, <0.9"] -risingwave = ["sqlalchemy-risingwave"] +# No release of sqlalchemy-risingwave has ever supported both SQLAlchemy 1.4 +# and 2.0 (version numbers don't track SQLAlchemy compat monotonically); pin +# to the newest 1.4-only release for now. Bump to >=2.0.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. +risingwave = ["sqlalchemy-risingwave>=1.4.1, <2.0.0"] shillelagh = ["shillelagh[all]>=1.4.4, <2"] singlestore = ["sqlalchemy-singlestoredb>=1.2.1, <2"] snowflake = ["snowflake-sqlalchemy>=1.11.0, <2"] diff --git a/requirements/base.txt b/requirements/base.txt index e22e3c7d8646..78550b9bf8d7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -150,6 +150,7 @@ flask-session==0.8.0 # via apache-superset (pyproject.toml) flask-sqlalchemy==2.5.1 # via + # apache-superset (pyproject.toml) # flask-appbuilder # flask-migrate flask-talisman==1.1.0 diff --git a/requirements/development.txt b/requirements/development.txt index 54caccc08c64..b6da9f5fb6a1 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -312,6 +312,7 @@ flask-session==0.8.0 flask-sqlalchemy==2.5.1 # via # -c requirements/base-constraint.txt + # apache-superset # flask-appbuilder # flask-migrate flask-talisman==1.1.0 @@ -989,7 +990,7 @@ sqlalchemy==1.4.54 # sqlalchemy-bigquery # sqlalchemy-continuum # sqlalchemy-utils -sqlalchemy-bigquery==1.17.0 +sqlalchemy-bigquery==1.17.1 # via apache-superset sqlalchemy-continuum==1.7.0 # via diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 3c8d7d488c89..1a99c397d7a1 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -275,6 +275,36 @@ class CompatibleDatabase(TypedDict, total=False): notes: str docs_url: str categories: list[str] # Override parent categories (e.g., for HOSTED_OPEN_SOURCE) + known_incompatibilities: list[KnownIncompatibility] + + +class KnownIncompatibility(TypedDict, total=False): + """A known, currently-unresolved incompatibility with a Superset dependency.""" + + dependency: str # e.g. "SQLAlchemy 2.0" + reason: str + tracking_url: str # upstream issue/PR tracking a fix, if one exists + since: str # ISO date this was last confirmed still broken + + +# Shared `known_incompatibilities` entry for the Aurora Data API driver +# (`sqlalchemy-aurora-data-api`), used by both the MySQL and PostgreSQL +# `compatible_databases` metadata for their respective Aurora entries. +AURORA_DATA_API_KNOWN_INCOMPATIBILITIES: list[KnownIncompatibility] = [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " + "dormant since 2021) nor the more active community fork " + "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " + "SQLAlchemy 2.0 compatibility." + ), + "tracking_url": ( + "https://github.com/cloud-utils/sqlalchemy-aurora-data-api/issues/43" + ), + "since": "2026-07-28", + } +] class DBEngineSpecMetadata(TypedDict, total=False): @@ -317,6 +347,11 @@ class DBEngineSpecMetadata(TypedDict, total=False): install_instructions: str version_requirements: str + # Known, currently-unresolved incompatibilities with a Superset + # dependency (e.g. a driver that doesn't yet support SQLAlchemy 2.0). + # Hopefully temporary; remove the entry once resolved upstream. + known_incompatibilities: list[KnownIncompatibility] + # Related databases (e.g., PostgreSQL-compatible databases) compatible_databases: list[CompatibleDatabase] diff --git a/superset/db_engine_specs/d1.py b/superset/db_engine_specs/d1.py index d49c0db2c625..a82a476f2b26 100644 --- a/superset/db_engine_specs/d1.py +++ b/superset/db_engine_specs/d1.py @@ -48,4 +48,16 @@ class CloudflareD1EngineSpec(SqliteEngineSpec): "cloudflare_d1_database_id": "D1 database ID", }, "install_instructions": "pip install superset-engine-d1", + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "sqlalchemy-d1 is very young (single release, Nov 2025) " + "and its only release pins sqlalchemy<2,>=1.4, " + "explicitly excluding SQLAlchemy 2.0." + ), + "tracking_url": ("https://github.com/sqlalchemy-cf-d1/sqlalchemy-d1"), + "since": "2026-07-28", + } + ], } diff --git a/superset/db_engine_specs/kusto.py b/superset/db_engine_specs/kusto.py index 85b563262bdd..179a891a9cce 100644 --- a/superset/db_engine_specs/kusto.py +++ b/superset/db_engine_specs/kusto.py @@ -131,6 +131,17 @@ class KustoSqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "notes": "Use native Kusto Query Language for advanced analytics.", }, ], + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "setup.py on the sqlalchemy-kusto main branch hard-pins " + "sqlalchemy==1.4.*; no SQLAlchemy 2.0 work has started." + ), + "tracking_url": "https://github.com/dodopizza/sqlalchemy-kusto", + "since": "2026-07-28", + } + ], } _time_grain_expressions = { diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 39d1b550cce6..459f9b847dd5 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -43,6 +43,7 @@ from superset.constants import TimeGrain from superset.db_engine_specs.base import ( + AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, BaseEngineSpec, BasicParametersMixin, DatabaseCategory, @@ -183,6 +184,7 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], + "known_incompatibilities": AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, }, ], } diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index 406e85effe27..870718268b64 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -34,6 +34,7 @@ from superset.constants import TimeGrain from superset.db_engine_specs.base import ( + AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, BaseEngineSpec, BasicParametersMixin, DatabaseCategory, @@ -545,6 +546,7 @@ class PostgresEngineSpec(BasicParametersMixin, PostgresBaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], + "known_incompatibilities": AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, }, ], } diff --git a/superset/db_engine_specs/solr.py b/superset/db_engine_specs/solr.py index 03dca6c2b660..eceec0efba70 100644 --- a/superset/db_engine_specs/solr.py +++ b/superset/db_engine_specs/solr.py @@ -43,6 +43,17 @@ class SolrEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "[/?use_ssl=true|false]" ), "default_port": 8983, + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "sqlalchemy-solr hard-pins sqlalchemy~=1.4.7 and has seen no " + "activity beyond dependabot bumps since 2024." + ), + "tracking_url": "https://github.com/aadel/sqlalchemy-solr", + "since": "2026-07-28", + } + ], } _time_grain_expressions = {