From b2eaee349e0668572324354426e2afffa3266127 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 28 Jul 2026 10:11:56 -0700 Subject: [PATCH 1/5] chore(sqla2): bump SQLAlchemy-dependent drivers to dual-compat versions Prep work for the SQLAlchemy 1.4 -> 2.0 core bump (discussion #40273, step 6). Every SQLAlchemy-dependent package Superset ships or offers as an optional extra gets bumped to the newest release that supports both SQLAlchemy 1.4 and 2.0, so each bump can be merged and tested independently under today's SQLAlchemy 1.4 pin before the core bump lands as its own final PR. - flask-sqlalchemy: pin explicitly (was purely transitive via flask-appbuilder, resolving to 2.5.1, 1.4-only) at >=3.0.5,<3.1, the last release before 3.1.0 drops 1.4 support. flask-appbuilder and marshmallow-sqlalchemy are already dual-compatible as pinned, no change needed. - sqlalchemy-bigquery: 1.17.0 -> 1.17.1 (patch; the project was archived 2026-05-16, likely its final release). - sqlalchemy-cratedb: 0.41.0 -> 0.43.1. - sqlalchemy-drill: drop an artificial <2 SQLAlchemy-version ceiling upstream never had (1.1.10 already supports 2.0 via import_dbapi). - firebolt-sqlalchemy: 1.0.0 -> 1.1.2. - sqlalchemy-ocient: 1.0.0 -> 3.0.0 (lower confidence: closed-source vendor package, no public changelog to verify against). - snowflake-sqlalchemy floor was already bumped independently by dependabot (#42418) to the same target this research landed on. Five extras (risingwave, exasol, firebird, redshift, dremio) have NO release supporting both SQLAlchemy versions -- version numbers cut straight from 1.4-only to 2.0-only with no overlap. Pinned each to its last 1.4-only version with a comment noting the 2.0 target to bump to once the core sqlalchemy bump lands, not before. Four extras have no SQLAlchemy 2.0 support anywhere in their ecosystem today and are flagged via a new `known_incompatibilities` metadata field (surfaced through the generated database docs): kusto, aurora-data-api (including our own dormant preset-io fork), solr, and d1. sqlalchemy-vertica-python was initially suspected to be a fifth, but turned out to already be dual-compatible (v0.6.3 already ships `import_dbapi`/`text()` wrapping) once the actual merged PR was checked instead of relying on PyPI metadata alone. Verified locally: full dependency resolution succeeds, the app boots and creates all tables cleanly under SQLAlchemy 1.4.54 + Flask-SQLAlchemy 3.0.5, 483/487 relevant unit tests pass (the 4 failures are a pre-existing local mysqlclient linkage issue, unrelated to this change), and the new metadata flows through generate_yaml_docs() end-to-end for all four flagged engines. --- pyproject.toml | 64 ++++++++++++++++++++++++---- requirements/base.txt | 3 +- requirements/development.txt | 5 ++- superset/db_engine_specs/base.py | 15 +++++++ superset/db_engine_specs/d1.py | 11 +++++ superset/db_engine_specs/kusto.py | 11 +++++ superset/db_engine_specs/mysql.py | 16 +++++++ superset/db_engine_specs/postgres.py | 16 +++++++ superset/db_engine_specs/solr.py | 11 +++++ 9 files changed, 140 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 118a1fe2dcc7..5474f9198d04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,11 @@ dependencies = [ "flask-login>=0.6.0, < 1.0", "flask-migrate>=4.1.0, <5.0", "flask-session>=0.4.0, <1.0", + # Transitive via flask-appbuilder, pinned explicitly to the last release + # that supports both SQLAlchemy 1.4 and 2.0 (3.1.0 drops 1.4). Prep for + # the SQLAlchemy 2.0 migration (see discussion #40273); step 6 will lift + # this pin once the core sqlalchemy bump lands. + "flask-sqlalchemy>=3.0.5, <3.1", "flask-wtf>=1.3.0, <2.0", "geopy", "greenlet<=3.5.4, >=3.5.4", @@ -123,15 +128,25 @@ dependencies = [ [project.optional-dependencies] athena = ["pyathena[pandas]>=3.35.2, <4"] +# No SQLAlchemy 2.0 support anywhere in this dialect's ecosystem today: our +# own preset-io/sqlalchemy-aurora-data-api fork is dead since 2021, and the +# more active community fork (cloud-utils/sqlalchemy-aurora-data-api) has an +# unresolved SQLAlchemy 2.0 break (upstream issue #43). See +# superset/db_engine_specs/aurora.py's known_incompatibilities metadata. aurora-data-api = ["preset-sqlalchemy-aurora-data-api>=0.2.8,<0.3"] bigquery = [ "pandas-gbq>=0.35.0", - "sqlalchemy-bigquery>=1.17.0", + # 1.17.1 is likely the final release: googleapis/python-bigquery-sqlalchemy + # was archived 2026-05-16. Both 1.17.0 and 1.17.1 support SQLAlchemy 1.4/2.0. + "sqlalchemy-bigquery>=1.17.1", "google-cloud-bigquery>=3.42.2", ] clickhouse = ["clickhouse-connect>=1.6.0, <2.0"] cockroachdb = ["cockroachdb>=0.3.5, <0.4"] -crate = ["sqlalchemy-cratedb>=0.41.0, <1"] +crate = ["sqlalchemy-cratedb>=0.43.1, <1"] +# sqlalchemy-d1 is very young (single release, Nov 2025); no SQLAlchemy 2.0 +# signal either way. See superset/db_engine_specs/d1.py's +# known_incompatibilities metadata. d1 = [ "superset-engine-d1>=0.1.0", "sqlalchemy-d1>=0.1.0", @@ -145,14 +160,27 @@ databricks = [ datafusion = ["flightsql-dbapi>=0.2.2, <0.3"] db2 = ["ibm-db-sa<=0.4.4, >=0.4.4"] denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"] -dremio = ["sqlalchemy-dremio>=1.2.1, <4"] -drill = ["sqlalchemy-drill>=1.1.10, <2"] +# sqlalchemy-dremio 3.0.5+ hard-pins sqlalchemy~=2.0.41, dropping 1.4; 3.0.4 +# is the last dual-compat release. Capped below 3.0.5 for now; widen back to +# <4 in lockstep with Superset's own SQLAlchemy 2.0 core bump (discussion +# #40273), not before. +dremio = ["sqlalchemy-dremio>=1.2.1, <3.0.5"] +# <2 was an artificial ceiling; upstream has no SQLAlchemy version cap and +# 1.1.10 already supports SQLAlchemy 2.0 (added `import_dbapi` in 1.1.7). +drill = ["sqlalchemy-drill>=1.1.10, <3"] druid = ["pydruid>=0.6.5,<0.7"] duckdb = ["duckdb>=1.5.4,<2", "duckdb-engine>=0.17.0"] dynamodb = ["pydynamodb>=0.8.2"] +# Effectively unmaintained (only dependabot bumps since 2024); hard-pinned to +# SQLAlchemy ~1.4.7 upstream, no SQLAlchemy 2.0 work. See +# superset/db_engine_specs/solr.py's known_incompatibilities metadata. solr = ["sqlalchemy-solr >= 0.2.4.3"] elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"] -exasol = ["sqlalchemy-exasol>=2.4.0, <8.0"] +# sqlalchemy-exasol cuts hard from SQLAlchemy 1.4-only (<6.0.0) to 2.0-only +# (>=6.0.0) with no dual-compat release. Capped below 6.0.0 for now; bump to +# >=6.0.0,<8.0 in lockstep with Superset's own SQLAlchemy 2.0 core bump +# (discussion #40273), not before. +exasol = ["sqlalchemy-exasol>=2.4.0, <6.0.0"] excel = ["xlrd>=2.0.2, <2.1"] # Async dashboard "Export Data/Images to Excel": uploads the workbook to S3 and # emails a pre-signed link. boto3 is imported lazily by superset.utils.s3, so @@ -165,8 +193,12 @@ fastmcp = [ # heuristic that under-counts JSON-heavy MCP responses. "tiktoken>=0.13.0,<1.0", ] -firebird = ["sqlalchemy-firebird>=0.8.0, <2.2"] -firebolt = ["firebolt-sqlalchemy>=1.0.0, <2"] +# sqlalchemy-firebird >=2.0.0 unconditionally requires SQLAlchemy 2.0 on +# Python >=3.8 (which covers Superset's >=3.11 floor), with no dual-compat +# release. Capped below 2.0.0 for now; bump to >=2.2.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. +firebird = ["sqlalchemy-firebird>=0.8.0, <2.0.0"] +firebolt = ["firebolt-sqlalchemy>=1.1.2, <2"] gevent = ["gevent>=26.4.0"] gsheets = ["shillelagh[gsheetsapi]>=1.4.4, <2"] hana = ["hdbcli==2.29.25", "sqlalchemy_hana==3.0.3"] @@ -177,6 +209,9 @@ hive = [ "thrift_sasl>=0.4.3, < 1.0.0", ] impala = ["impyla>=0.24.0, <0.25"] +# Actively maintained upstream, but setup.py on main hard-pins +# sqlalchemy==1.4.*, no SQLAlchemy 2.0 work yet. See +# superset/db_engine_specs/kusto.py's known_incompatibilities metadata. kusto = ["sqlalchemy-kusto>=3.1.2, <4"] kylin = ["kylinpy>=2.8.4, <2.9"] mssql = ["pymssql>=2.3.13, <3"] @@ -184,7 +219,10 @@ mssql = ["pymssql>=2.3.13, <3"] motherduck = ["apache-superset[duckdb]"] mysql = ["mysqlclient>=2.2.8, <3"] ocient = [ - "sqlalchemy-ocient>=1.0.0, <4", + # Closed-source vendor package with no public changelog; permissive + # unpinned sqlalchemy>=1.4 declared, but SQLAlchemy 2.0 support is + # unverified. Lower confidence than the other bumps in this PR. + "sqlalchemy-ocient>=3.0.0, <4", "pyocient>=1.0.15, <4", "shapely", "geojson", @@ -197,8 +235,16 @@ postgres = ["psycopg2-binary==2.9.12"] presto = ["pyhive[presto]>=0.6.5"] trino = ["trino>=0.338.0"] prophet = ["prophet>=1.3.0, <2"] +# sqlalchemy-redshift cuts hard from SQLAlchemy 1.4-only (0.8.x) to 2.0-only +# (>=1.0.0) with no dual-compat release; the existing <0.9 ceiling already +# keeps this on the 1.4-only line. Bump to >=1.0.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. redshift = ["sqlalchemy-redshift>=0.8.1, <0.9"] -risingwave = ["sqlalchemy-risingwave"] +# No release of sqlalchemy-risingwave has ever supported both SQLAlchemy 1.4 +# and 2.0 (version numbers don't track SQLAlchemy compat monotonically); pin +# to the newest 1.4-only release for now. Bump to >=2.0.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. +risingwave = ["sqlalchemy-risingwave>=1.4.1, <2.0.0"] shillelagh = ["shillelagh[all]>=1.4.4, <2"] singlestore = ["sqlalchemy-singlestoredb>=1.2.1, <2"] snowflake = ["snowflake-sqlalchemy>=1.11.0, <2"] diff --git a/requirements/base.txt b/requirements/base.txt index e22e3c7d8646..9fa8290472c6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -148,8 +148,9 @@ flask-migrate==4.1.0 # via apache-superset (pyproject.toml) flask-session==0.8.0 # via apache-superset (pyproject.toml) -flask-sqlalchemy==2.5.1 +flask-sqlalchemy==3.0.5 # via + # apache-superset (pyproject.toml) # flask-appbuilder # flask-migrate flask-talisman==1.1.0 diff --git a/requirements/development.txt b/requirements/development.txt index 54caccc08c64..639925b72c77 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -309,9 +309,10 @@ flask-session==0.8.0 # via # -c requirements/base-constraint.txt # apache-superset -flask-sqlalchemy==2.5.1 +flask-sqlalchemy==3.0.5 # via # -c requirements/base-constraint.txt + # apache-superset # flask-appbuilder # flask-migrate flask-talisman==1.1.0 @@ -989,7 +990,7 @@ sqlalchemy==1.4.54 # sqlalchemy-bigquery # sqlalchemy-continuum # sqlalchemy-utils -sqlalchemy-bigquery==1.17.0 +sqlalchemy-bigquery==1.17.1 # via apache-superset sqlalchemy-continuum==1.7.0 # via diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 3c8d7d488c89..6b609a7ad30d 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -275,6 +275,16 @@ class CompatibleDatabase(TypedDict, total=False): notes: str docs_url: str categories: list[str] # Override parent categories (e.g., for HOSTED_OPEN_SOURCE) + known_incompatibilities: list[KnownIncompatibility] + + +class KnownIncompatibility(TypedDict, total=False): + """A known, currently-unresolved incompatibility with a Superset dependency.""" + + dependency: str # e.g. "SQLAlchemy 2.0" + reason: str + tracking_url: str # upstream issue/PR tracking a fix, if one exists + since: str # ISO date this was last confirmed still broken class DBEngineSpecMetadata(TypedDict, total=False): @@ -317,6 +327,11 @@ class DBEngineSpecMetadata(TypedDict, total=False): install_instructions: str version_requirements: str + # Known, currently-unresolved incompatibilities with a Superset + # dependency (e.g. a driver that doesn't yet support SQLAlchemy 2.0). + # Hopefully temporary; remove the entry once resolved upstream. + known_incompatibilities: list[KnownIncompatibility] + # Related databases (e.g., PostgreSQL-compatible databases) compatible_databases: list[CompatibleDatabase] diff --git a/superset/db_engine_specs/d1.py b/superset/db_engine_specs/d1.py index d49c0db2c625..40214bf606a5 100644 --- a/superset/db_engine_specs/d1.py +++ b/superset/db_engine_specs/d1.py @@ -48,4 +48,15 @@ class CloudflareD1EngineSpec(SqliteEngineSpec): "cloudflare_d1_database_id": "D1 database ID", }, "install_instructions": "pip install superset-engine-d1", + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "sqlalchemy-d1 is very young (single release, Nov 2025); " + "no confirmed SQLAlchemy 2.0 support either way yet." + ), + "tracking_url": ("https://github.com/sqlalchemy-cf-d1/sqlalchemy-d1"), + "since": "2026-07-28", + } + ], } diff --git a/superset/db_engine_specs/kusto.py b/superset/db_engine_specs/kusto.py index 85b563262bdd..179a891a9cce 100644 --- a/superset/db_engine_specs/kusto.py +++ b/superset/db_engine_specs/kusto.py @@ -131,6 +131,17 @@ class KustoSqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "notes": "Use native Kusto Query Language for advanced analytics.", }, ], + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "setup.py on the sqlalchemy-kusto main branch hard-pins " + "sqlalchemy==1.4.*; no SQLAlchemy 2.0 work has started." + ), + "tracking_url": "https://github.com/dodopizza/sqlalchemy-kusto", + "since": "2026-07-28", + } + ], } _time_grain_expressions = { diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 39d1b550cce6..766107e3dc71 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -183,6 +183,22 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " + "dormant since 2021) nor the more active community fork " + "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " + "SQLAlchemy 2.0 compatibility." + ), + "tracking_url": ( + "https://github.com/cloud-utils/" + "sqlalchemy-aurora-data-api/issues/43" + ), + "since": "2026-07-28", + } + ], }, ], } diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index 406e85effe27..a6b4339e0a53 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -545,6 +545,22 @@ class PostgresEngineSpec(BasicParametersMixin, PostgresBaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " + "dormant since 2021) nor the more active community fork " + "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " + "SQLAlchemy 2.0 compatibility." + ), + "tracking_url": ( + "https://github.com/cloud-utils/" + "sqlalchemy-aurora-data-api/issues/43" + ), + "since": "2026-07-28", + } + ], }, ], } diff --git a/superset/db_engine_specs/solr.py b/superset/db_engine_specs/solr.py index 03dca6c2b660..eceec0efba70 100644 --- a/superset/db_engine_specs/solr.py +++ b/superset/db_engine_specs/solr.py @@ -43,6 +43,17 @@ class SolrEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "[/?use_ssl=true|false]" ), "default_port": 8983, + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "sqlalchemy-solr hard-pins sqlalchemy~=1.4.7 and has seen no " + "activity beyond dependabot bumps since 2024." + ), + "tracking_url": "https://github.com/aadel/sqlalchemy-solr", + "since": "2026-07-28", + } + ], } _time_grain_expressions = { From de106d425be24bf06a8226e8aec1a2a66ae1e6d4 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 28 Jul 2026 11:01:36 -0700 Subject: [PATCH 2/5] fix(tests): restore NullPool for SQLite under Flask-SQLAlchemy 3.0.5 Flask-SQLAlchemy 3.0 stopped defaulting SQLite engines to NullPool when pool_size isn't set (changelog: "SQLite engines do not use NullPool if pool_size is 0"). Superset never configures pool_size, so under 2.5.1 this was silently applying; under 3.0.5 it falls back to QueuePool instead, which holds connections open. With a file-based SQLite db shared between the test process and the Celery worker, and ENABLE_VERSIONING_CAPTURE adding a version_transaction write to every commit, held-open pooled connections caused a wave of "database is locked" errors in test-sqlite CI (19 failed, 47 errors) that don't reproduce on master. --- tests/integration_tests/superset_test_config.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration_tests/superset_test_config.py b/tests/integration_tests/superset_test_config.py index b1d786f9957b..9efa5e275827 100644 --- a/tests/integration_tests/superset_test_config.py +++ b/tests/integration_tests/superset_test_config.py @@ -21,6 +21,7 @@ from datetime import timedelta from sqlalchemy.engine import make_url +from sqlalchemy.pool import NullPool from superset.config import * # noqa: F403 from superset.config import DATA_DIR @@ -58,6 +59,19 @@ "SQLite Database support for metadata databases will be " "removed in a future version of Superset." ) + # Flask-SQLAlchemy 3.0 stopped defaulting SQLite engines to NullPool when + # pool_size isn't set (it now falls back to QueuePool, same as any other + # backend). With a file-based SQLite db shared between this test process + # and the Celery worker, pooled/held-open connections cause "database is + # locked" errors under concurrent writes -- especially with + # ENABLE_VERSIONING_CAPTURE on, which adds a version_transaction write to + # every commit. NullPool restores the old (correct, for this use case) + # behavior: every checkout opens a fresh connection, every checkin closes + # it, so no connection sits idle holding a lock. + SQLALCHEMY_ENGINE_OPTIONS = { # noqa: F405 + **SQLALCHEMY_ENGINE_OPTIONS, # noqa: F405 + "poolclass": NullPool, + } # Speeding up the tests.integration_tests. PRESTO_POLL_INTERVAL = 0.1 From 3bcf4e86676893970bda2359ef3613be8a005e08 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 28 Jul 2026 11:29:53 -0700 Subject: [PATCH 3/5] revert(flask-sqlalchemy): back to 2.5.1, cap explicitly below 3.0 Real CI (test-sqlite, test-mysql) surfaced a structural incompatibility between Flask-SQLAlchemy 3.0.5's session/app-context scoping and how Superset currently crosses context boundaries (Celery tasks, background jobs): widespread "NoneType has no attribute X" failures across unrelated test files (dashboards, embedded views, security, tasks) plus MySQL "Lock wait timeout exceeded" errors. Tried restoring NullPool for SQLite (Flask-SQLAlchemy 3.0 stopped defaulting SQLite to NullPool when pool_size is unset) since that matched the initial "database is locked" symptom -- it had zero effect (identical failure count with and without it), confirming the locking was a downstream symptom of the deeper session-scoping issue, not a connection-pool problem. Reverting that fix along with the bump. Without an explicit upper bound, uv's resolver naturally re-picks 3.0.5 on any lockfile regen (flask-appbuilder allows <4), so this pins <3.0 explicitly rather than just removing the forcing pin. This needs dedicated investigation into Superset's session/app-context handling, not a bump slipped into driver-compat prep. Revisit alongside the actual SQLAlchemy 2.0 core bump (discussion #40273, step 6) -- Flask-SQLAlchemy 3.1+ requires SQLAlchemy 2.0 anyway, so this was always going to need to happen around the same time. --- pyproject.toml | 14 +++++++++----- requirements/base.txt | 2 +- requirements/development.txt | 2 +- tests/integration_tests/superset_test_config.py | 14 -------------- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5474f9198d04..21fb60d1088c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,11 +60,15 @@ dependencies = [ "flask-login>=0.6.0, < 1.0", "flask-migrate>=4.1.0, <5.0", "flask-session>=0.4.0, <1.0", - # Transitive via flask-appbuilder, pinned explicitly to the last release - # that supports both SQLAlchemy 1.4 and 2.0 (3.1.0 drops 1.4). Prep for - # the SQLAlchemy 2.0 migration (see discussion #40273); step 6 will lift - # this pin once the core sqlalchemy bump lands. - "flask-sqlalchemy>=3.0.5, <3.1", + # Pinned explicitly below 3.0: 3.0.5 resolves without conflict and + # supports both SQLAlchemy 1.4 and 2.0, but real CI runs surfaced a + # structural incompatibility with Superset's current session/app-context + # handling across Celery task boundaries (see PR #42542) -- widespread + # "NoneType has no attribute X" failures and MySQL lock-wait timeouts, + # not just a connection-pool quirk. Needs dedicated investigation, not a + # driver-compat-prep bump; revisit alongside the actual SQLAlchemy 2.0 + # core bump (discussion #40273, step 6). + "flask-sqlalchemy>=2.5.1, <3.0", "flask-wtf>=1.3.0, <2.0", "geopy", "greenlet<=3.5.4, >=3.5.4", diff --git a/requirements/base.txt b/requirements/base.txt index 9fa8290472c6..78550b9bf8d7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -148,7 +148,7 @@ flask-migrate==4.1.0 # via apache-superset (pyproject.toml) flask-session==0.8.0 # via apache-superset (pyproject.toml) -flask-sqlalchemy==3.0.5 +flask-sqlalchemy==2.5.1 # via # apache-superset (pyproject.toml) # flask-appbuilder diff --git a/requirements/development.txt b/requirements/development.txt index 639925b72c77..b6da9f5fb6a1 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -309,7 +309,7 @@ flask-session==0.8.0 # via # -c requirements/base-constraint.txt # apache-superset -flask-sqlalchemy==3.0.5 +flask-sqlalchemy==2.5.1 # via # -c requirements/base-constraint.txt # apache-superset diff --git a/tests/integration_tests/superset_test_config.py b/tests/integration_tests/superset_test_config.py index 9efa5e275827..b1d786f9957b 100644 --- a/tests/integration_tests/superset_test_config.py +++ b/tests/integration_tests/superset_test_config.py @@ -21,7 +21,6 @@ from datetime import timedelta from sqlalchemy.engine import make_url -from sqlalchemy.pool import NullPool from superset.config import * # noqa: F403 from superset.config import DATA_DIR @@ -59,19 +58,6 @@ "SQLite Database support for metadata databases will be " "removed in a future version of Superset." ) - # Flask-SQLAlchemy 3.0 stopped defaulting SQLite engines to NullPool when - # pool_size isn't set (it now falls back to QueuePool, same as any other - # backend). With a file-based SQLite db shared between this test process - # and the Celery worker, pooled/held-open connections cause "database is - # locked" errors under concurrent writes -- especially with - # ENABLE_VERSIONING_CAPTURE on, which adds a version_transaction write to - # every commit. NullPool restores the old (correct, for this use case) - # behavior: every checkout opens a fresh connection, every checkin closes - # it, so no connection sits idle holding a lock. - SQLALCHEMY_ENGINE_OPTIONS = { # noqa: F405 - **SQLALCHEMY_ENGINE_OPTIONS, # noqa: F405 - "poolclass": NullPool, - } # Speeding up the tests.integration_tests. PRESTO_POLL_INTERVAL = 0.1 From ae068d630dcd108acb2bb7926dac9155cd62ca8d Mon Sep 17 00:00:00 2001 From: rusackas Date: Tue, 28 Jul 2026 15:32:02 -0700 Subject: [PATCH 4/5] fix: dedupe Aurora Data API incompatibility metadata, normalize solr pin Extract the duplicated Aurora Data API known_incompatibilities block from mysql.py/postgres.py into a shared base.py constant, and drop the stray space in the sqlalchemy-solr version pin for PEP 508 consistency. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 2 +- superset/db_engine_specs/base.py | 20 ++++++++++++++++++++ superset/db_engine_specs/mysql.py | 18 ++---------------- superset/db_engine_specs/postgres.py | 18 ++---------------- 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 21fb60d1088c..a5c4b8978eba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,7 +178,7 @@ dynamodb = ["pydynamodb>=0.8.2"] # Effectively unmaintained (only dependabot bumps since 2024); hard-pinned to # SQLAlchemy ~1.4.7 upstream, no SQLAlchemy 2.0 work. See # superset/db_engine_specs/solr.py's known_incompatibilities metadata. -solr = ["sqlalchemy-solr >= 0.2.4.3"] +solr = ["sqlalchemy-solr>=0.2.4.3"] elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"] # sqlalchemy-exasol cuts hard from SQLAlchemy 1.4-only (<6.0.0) to 2.0-only # (>=6.0.0) with no dual-compat release. Capped below 6.0.0 for now; bump to diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 6b609a7ad30d..1a99c397d7a1 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -287,6 +287,26 @@ class KnownIncompatibility(TypedDict, total=False): since: str # ISO date this was last confirmed still broken +# Shared `known_incompatibilities` entry for the Aurora Data API driver +# (`sqlalchemy-aurora-data-api`), used by both the MySQL and PostgreSQL +# `compatible_databases` metadata for their respective Aurora entries. +AURORA_DATA_API_KNOWN_INCOMPATIBILITIES: list[KnownIncompatibility] = [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " + "dormant since 2021) nor the more active community fork " + "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " + "SQLAlchemy 2.0 compatibility." + ), + "tracking_url": ( + "https://github.com/cloud-utils/sqlalchemy-aurora-data-api/issues/43" + ), + "since": "2026-07-28", + } +] + + class DBEngineSpecMetadata(TypedDict, total=False): """ Metadata for database engine documentation and UI display. diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 766107e3dc71..459f9b847dd5 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -43,6 +43,7 @@ from superset.constants import TimeGrain from superset.db_engine_specs.base import ( + AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, BaseEngineSpec, BasicParametersMixin, DatabaseCategory, @@ -183,22 +184,7 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], - "known_incompatibilities": [ - { - "dependency": "SQLAlchemy 2.0", - "reason": ( - "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " - "dormant since 2021) nor the more active community fork " - "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " - "SQLAlchemy 2.0 compatibility." - ), - "tracking_url": ( - "https://github.com/cloud-utils/" - "sqlalchemy-aurora-data-api/issues/43" - ), - "since": "2026-07-28", - } - ], + "known_incompatibilities": AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, }, ], } diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index a6b4339e0a53..870718268b64 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -34,6 +34,7 @@ from superset.constants import TimeGrain from superset.db_engine_specs.base import ( + AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, BaseEngineSpec, BasicParametersMixin, DatabaseCategory, @@ -545,22 +546,7 @@ class PostgresEngineSpec(BasicParametersMixin, PostgresBaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], - "known_incompatibilities": [ - { - "dependency": "SQLAlchemy 2.0", - "reason": ( - "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " - "dormant since 2021) nor the more active community fork " - "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " - "SQLAlchemy 2.0 compatibility." - ), - "tracking_url": ( - "https://github.com/cloud-utils/" - "sqlalchemy-aurora-data-api/issues/43" - ), - "since": "2026-07-28", - } - ], + "known_incompatibilities": AURORA_DATA_API_KNOWN_INCOMPATIBILITIES, }, ], } From 7045efccb6de7e790cf3af343364e7c38519a4a2 Mon Sep 17 00:00:00 2001 From: rusackas Date: Fri, 31 Jul 2026 16:07:41 -0700 Subject: [PATCH 5/5] fix: render known_incompatibilities in docs, resolve constants in AST fallback, correct d1 SQLAlchemy 2.0 claim - Add the TypeScript contract for `known_incompatibilities` and render it on the generated database docs pages (both top-level and within each compatible-database entry), so the field added for the Aurora Data API / D1 SQLAlchemy 2.0 warnings is actually visible instead of just reaching the JSON unused. - Teach the AST-based docs fallback extractor to resolve module-level literal constants (e.g. AURORA_DATA_API_KNOWN_INCOMPATIBILITIES) so a metadata dict that references a shared constant by name no longer serializes as the bare identifier string when Flask context isn't available. - Correct the d1.py known_incompatibilities reason: sqlalchemy-d1's only release (0.1.0) pins `sqlalchemy<2,>=1.4`, which explicitly excludes SQLAlchemy 2.0 rather than merely lacking confirmation either way. Co-Authored-By: Claude Opus 4.8 --- docs/scripts/generate-database-docs.mjs | 55 +++++++++++++-- .../src/components/databases/DatabasePage.tsx | 70 ++++++++++++++++++- docs/src/components/databases/types.ts | 9 +++ pyproject.toml | 4 +- superset/db_engine_specs/d1.py | 5 +- 5 files changed, 131 insertions(+), 12 deletions(-) diff --git a/docs/scripts/generate-database-docs.mjs b/docs/scripts/generate-database-docs.mjs index 25c6e0784579..e43b4d487e8e 100644 --- a/docs/scripts/generate-database-docs.mjs +++ b/docs/scripts/generate-database-docs.mjs @@ -101,21 +101,29 @@ import json import ast import os -def eval_node(node): - """Safely evaluate an AST node as a Python literal.""" +def eval_node(node, constants=None): + """Safely evaluate an AST node as a Python literal. + + \`constants\` is an optional dict of module-level constant names -> already + -resolved Python values. It lets us resolve references like + \`AURORA_DATA_API_KNOWN_INCOMPATIBILITIES\` that point at a list/dict + defined (and potentially imported across files) elsewhere in + db_engine_specs, instead of falling through to the bare identifier + string. + """ if node is None: return None if isinstance(node, ast.Constant): return node.value elif isinstance(node, ast.List): - return [eval_node(e) for e in node.elts] + return [eval_node(e, constants) for e in node.elts] elif isinstance(node, ast.Dict): result = {} for k, v in zip(node.keys, node.values): if k is not None: - key = eval_node(k) + key = eval_node(k, constants) if key is not None: - result[key] = eval_node(v) + result[key] = eval_node(v, constants) return result elif isinstance(node, ast.Name): # Handle True, False, None constants @@ -125,12 +133,14 @@ def eval_node(node): return False elif node.id == 'None': return None + if constants and node.id in constants: + return constants[node.id] return node.id elif isinstance(node, ast.Attribute): # Handle DatabaseCategory.SOMETHING - return just the attribute name return node.attr elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): - left, right = eval_node(node.left), eval_node(node.right) + left, right = eval_node(node.left, constants), eval_node(node.right, constants) if isinstance(left, str) and isinstance(right, str): return left + right return None @@ -274,6 +284,37 @@ CAP_METHODS = { # Intermediate base classes (e.g. PrestoBaseEngineSpec) do count as overrides. TRUE_BASE_CLASS = 'BaseEngineSpec' +# Pass 0: collect module-level literal constants across every engine spec +# file (e.g. AURORA_DATA_API_KNOWN_INCOMPATIBILITIES in base.py, imported +# into mysql.py's \`compatible_databases\` metadata) so \`metadata\` dicts +# that reference a shared constant by name resolve to its actual value +# instead of the bare identifier string. Only module-scope assignments +# (tree.body, not nested in classes/functions) are considered. +MODULE_CONSTANTS = {} +for filename in sorted(os.listdir(specs_dir)): + if not filename.endswith('.py') or filename in ('__init__.py', 'lib.py', 'lint_metadata.py'): + continue + filepath = os.path.join(specs_dir, filename) + try: + with open(filepath) as f: + source = f.read() + tree = ast.parse(source) + for item in tree.body: + targets = [] + if isinstance(item, ast.Assign): + targets = item.targets + elif isinstance(item, ast.AnnAssign) and item.value is not None: + # Handle annotated module-level constants, e.g. + # \`AURORA_DATA_API_KNOWN_INCOMPATIBILITIES: list[KnownIncompatibility] = [...]\` + targets = [item.target] + for target in targets: + if isinstance(target, ast.Name) and target.id.isupper(): + val = eval_node(item.value, MODULE_CONSTANTS) + if val is not None: + MODULE_CONSTANTS[target.id] = val + except Exception: + continue + # First pass: collect all class info (name, bases, metadata, cap_attrs, direct_methods) class_info = {} # class_name -> {bases: [], metadata: {}, engine_name: str, filename: str, ...} @@ -330,7 +371,7 @@ for filename in sorted(os.listdir(specs_dir)): if isinstance(val, str): engine_attr = val elif target.id == 'metadata': - metadata = eval_node(item.value) + metadata = eval_node(item.value, MODULE_CONSTANTS) elif target.id in CAP_ATTR_DEFAULTS: val = eval_node(item.value) if isinstance(val, bool): diff --git a/docs/src/components/databases/DatabasePage.tsx b/docs/src/components/databases/DatabasePage.tsx index 8e4b8e836680..a1f293e11d62 100644 --- a/docs/src/components/databases/DatabasePage.tsx +++ b/docs/src/components/databases/DatabasePage.tsx @@ -41,7 +41,7 @@ import { GithubOutlined, BugOutlined, } from '@ant-design/icons'; -import type { DatabaseInfo } from './types'; +import type { DatabaseInfo, KnownIncompatibility } from './types'; // Simple code block component for connection strings const CodeBlock: React.FC<{ children: React.ReactNode }> = ({ children }) => ( @@ -253,6 +253,53 @@ const DatabasePage: React.FC = ({ database, name }) => { ); }; + // Render known incompatibilities with a Superset dependency (e.g. a driver + // that doesn't yet support SQLAlchemy 2.0). Shared between the top-level + // documentation and each compatible-database entry. + const renderKnownIncompatibilities = ( + incompatibilities?: KnownIncompatibility[], + ) => { + if (!incompatibilities?.length) return null; + + return ( + + {incompatibilities.map((incompat, idx) => ( + + {incompat.reason && ( + + {incompat.reason} + + )} + + {incompat.tracking_url && ( + + Tracking issue + + )} + {incompat.since && ( + + Last confirmed: {incompat.since} + + )} + + + } + /> + ))} + + ); + }; + // Render compatible databases (for PostgreSQL, etc.) const renderCompatibleDatabases = () => { if (!docs?.compatible_databases?.length) return null; @@ -320,6 +367,16 @@ const DatabasePage: React.FC = ({ database, name }) => { /> )} + {compat.known_incompatibilities?.length > 0 && ( +
+ Known Incompatibilities: +
+ {renderKnownIncompatibilities( + compat.known_incompatibilities, + )} +
+
+ )} {compat.notes && ( = ({ database, name }) => { )} + {/* Known Incompatibilities */} + {docs?.known_incompatibilities?.length > 0 && ( + + {renderKnownIncompatibilities(docs.known_incompatibilities)} + + )} + {/* Installation */} {(docs?.pypi_packages?.length || docs?.install_instructions) && ( diff --git a/docs/src/components/databases/types.ts b/docs/src/components/databases/types.ts index 7b3f61a57c5f..fb3689e9bc1e 100644 --- a/docs/src/components/databases/types.ts +++ b/docs/src/components/databases/types.ts @@ -72,6 +72,13 @@ export interface SSLConfiguration { }; } +export interface KnownIncompatibility { + dependency: string; // e.g. "SQLAlchemy 2.0" + reason?: string; + tracking_url?: string; // upstream issue/PR tracking a fix, if one exists + since?: string; // ISO date this was last confirmed still broken +} + export interface CompatibleDatabase { name: string; description?: string; @@ -84,6 +91,7 @@ export interface CompatibleDatabase { connection_examples?: ConnectionExample[]; notes?: string; docs_url?: string; + known_incompatibilities?: KnownIncompatibility[]; } export interface CustomError { @@ -123,6 +131,7 @@ export interface DatabaseDocumentation { advanced_features?: Record; compatible_databases?: CompatibleDatabase[]; custom_errors?: CustomError[]; // Database-specific error messages and troubleshooting info + known_incompatibilities?: KnownIncompatibility[]; // Unresolved incompatibilities with a Superset dependency } export interface TimeGrains { diff --git a/pyproject.toml b/pyproject.toml index a5c4b8978eba..6ea0496b2fc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,8 +148,8 @@ bigquery = [ clickhouse = ["clickhouse-connect>=1.6.0, <2.0"] cockroachdb = ["cockroachdb>=0.3.5, <0.4"] crate = ["sqlalchemy-cratedb>=0.43.1, <1"] -# sqlalchemy-d1 is very young (single release, Nov 2025); no SQLAlchemy 2.0 -# signal either way. See superset/db_engine_specs/d1.py's +# sqlalchemy-d1's only release (0.1.0, Nov 2025) pins sqlalchemy<2,>=1.4, +# explicitly excluding SQLAlchemy 2.0. See superset/db_engine_specs/d1.py's # known_incompatibilities metadata. d1 = [ "superset-engine-d1>=0.1.0", diff --git a/superset/db_engine_specs/d1.py b/superset/db_engine_specs/d1.py index 40214bf606a5..a82a476f2b26 100644 --- a/superset/db_engine_specs/d1.py +++ b/superset/db_engine_specs/d1.py @@ -52,8 +52,9 @@ class CloudflareD1EngineSpec(SqliteEngineSpec): { "dependency": "SQLAlchemy 2.0", "reason": ( - "sqlalchemy-d1 is very young (single release, Nov 2025); " - "no confirmed SQLAlchemy 2.0 support either way yet." + "sqlalchemy-d1 is very young (single release, Nov 2025) " + "and its only release pins sqlalchemy<2,>=1.4, " + "explicitly excluding SQLAlchemy 2.0." ), "tracking_url": ("https://github.com/sqlalchemy-cf-d1/sqlalchemy-d1"), "since": "2026-07-28",