Skip to content

[CORE] Limit SparkSessionExtension suite to Gluten test #12475

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zml1206 merged 2 commits into from
Jul 9, 2026
Merged

[CORE] Limit SparkSessionExtension suite to Gluten test #12475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6292441
[VL] Fix GlutenSparkSessionExtensionSuite in Spark 4.x
zml1206 Jul 8, 2026
14d661f
Limit GlutenSparkSessionExtensionSuite to Gluten-specific test
zml1206 Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ...ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import org.apache.spark.sql.sources._
// scalastyle:off line.size.limit

class ClickHouseTestSettings extends BackendTestSettings {
import SuiteSettings._

// disable tests that will break the whole UT
override def shouldRun(suiteName: String, testName: String): Boolean = {
Expand Down Expand Up @@ -1881,6 +1882,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeGlutenTest("fallbackSummary with cache")
.excludeGlutenTest("fallbackSummary with cached data and shuffle")
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenHiveSQLQueryCHSuite]
enableSuite[GlutenPercentileSuite]

Expand Down
2 changes: 2 additions & 0 deletions gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import org.apache.spark.sql.sources._
// scalastyle:off line.size.limit

class VeloxTestSettings extends BackendTestSettings {
import SuiteSettings._
enableSuite[GlutenStringFunctionsSuite]
enableSuite[GlutenBloomFilterAggregateQuerySuite]
enableSuite[GlutenBloomFilterAggregateQuerySuiteCGOff]
Expand Down Expand Up @@ -910,6 +911,7 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenImplicitsTest]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenSQLCollectLimitExecSuite]
enableSuite[GlutenBatchEvalPythonExecSuite]
// Replaced with other tests that check for native operations
Expand Down
2 changes: 2 additions & 0 deletions ...ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import org.apache.spark.sql.sources._
// scalastyle:off line.size.limit

class ClickHouseTestSettings extends BackendTestSettings {
import SuiteSettings._

// disable tests that will break the whole UT
override def shouldRun(suiteName: String, testName: String): Boolean = {
Expand Down Expand Up @@ -1740,6 +1741,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
"SELECT structFieldSimple.key, arrayFieldSimple[1] FROM tableWithSchema a where int_Field=1")
.exclude("SELECT structFieldComplex.Value.`value_(2)` FROM tableWithSchema")
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenHiveSQLQueryCHSuite]
enableSuite[GlutenPercentileSuite]
enableSuite[GlutenTryCastSuite]
Expand Down
2 changes: 2 additions & 0 deletions gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import org.apache.spark.sql.sources.{GlutenBucketedReadWithoutHiveSupportSuite,
// scalastyle:off line.size.limit

class VeloxTestSettings extends BackendTestSettings {
import SuiteSettings._
enableSuite[GlutenStringFunctionsSuite]
enableSuite[GlutenBloomFilterAggregateQuerySuite]
enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite]
Expand Down Expand Up @@ -955,6 +956,7 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenWindowQuerySuite]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenGroupBasedDeleteFromTableSuite]
enableSuite[GlutenDeltaBasedDeleteFromTableSuite]
enableSuite[GlutenDataFrameToSchemaSuite]
Expand Down
2 changes: 2 additions & 0 deletions ...ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import org.apache.spark.sql.sources._
// scalastyle:off line.size.limit

class ClickHouseTestSettings extends BackendTestSettings {
import SuiteSettings._

enableSuite[ClickHouseAdaptiveQueryExecSuite]
.includeAllGlutenTests()
Expand Down Expand Up @@ -2100,6 +2101,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("sorting on YearMonthIntervalType(0,1) with nullable=false, sortOrder=List('a DESC NULLS LAST)")
.excludeCH("sorting on YearMonthIntervalType(0,1) with nullable=false, sortOrder=List('a DESC NULLS FIRST)")
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenStatisticsCollectionSuite]
// The output byte size of Velox is different
.includeCH("SPARK-33687: analyze all tables in a specific database")
Expand Down
2 changes: 2 additions & 0 deletions gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import org.apache.spark.sql.sources._
// scalastyle:off line.size.limit

class VeloxTestSettings extends BackendTestSettings {
import SuiteSettings._
enableSuite[GlutenStringFunctionsSuite]
enableSuite[GlutenBloomFilterAggregateQuerySuite]
enableSuite[GlutenBloomFilterAggregateQuerySuiteCGOff]
Expand Down Expand Up @@ -913,6 +914,7 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenWindowQuerySuite]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenGroupBasedDeleteFromTableSuite]
enableSuite[GlutenDeltaBasedDeleteFromTableSuite]
enableSuite[GlutenDataFrameToSchemaSuite]
Expand Down
2 changes: 2 additions & 0 deletions ...ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import org.apache.spark.sql.sources._
// scalastyle:off line.size.limit

class ClickHouseTestSettings extends BackendTestSettings {
import SuiteSettings._

enableSuite[ClickHouseAdaptiveQueryExecSuite]
.includeAllGlutenTests()
Expand Down Expand Up @@ -2073,6 +2074,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("sorting on YearMonthIntervalType(0,1) with nullable=false, sortOrder=List('a DESC NULLS LAST)")
.excludeCH("sorting on YearMonthIntervalType(0,1) with nullable=false, sortOrder=List('a DESC NULLS FIRST)")
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenStatisticsCollectionSuite]
// The output byte size of Velox is different
.includeCH("SPARK-33687: analyze all tables in a specific database")
Expand Down
4 changes: 3 additions & 1 deletion gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import org.apache.spark.sql.streaming._
// scalastyle:off line.size.limit

class VeloxTestSettings extends BackendTestSettings {
import SuiteSettings._
private val ansiNoFallback: Boolean =
sys.props.get(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key).contains("false")
enableSuite[GlutenStringFunctionsSuite]
Expand Down Expand Up @@ -1148,7 +1149,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenSQLQuerySuiteAE]
enableSuite[GlutenWindowQuerySuite]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
// TODO: 4.x enableSuite[GlutenSparkSessionExtensionSuite] // 1 failure
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenGroupBasedDeleteFromTableSuite]
enableSuite[GlutenDeltaBasedDeleteFromTableSuite]
enableSuite[GlutenDataFrameToSchemaSuite]
Expand Down
4 changes: 3 additions & 1 deletion gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ package org.apache.spark.sql

import org.apache.gluten.config.GlutenConfig

class GlutenSparkSessionExtensionSuite extends SparkSessionExtensionSuite with GlutenTestsTrait {
class GlutenSparkSessionExtensionSuite
extends SparkSessionExtensionSuite
with GlutenTestsCommonTrait {

testGluten("customColumnarOp") {
val extensions = DummyFilterColmnarHelper.create {
Expand Down
2 changes: 2 additions & 0 deletions ...ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import org.apache.spark.sql.sources._
// scalastyle:off line.size.limit

class ClickHouseTestSettings extends BackendTestSettings {
import SuiteSettings._

enableSuite[ClickHouseAdaptiveQueryExecSuite]
.includeAllGlutenTests()
Expand Down Expand Up @@ -2073,6 +2074,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("sorting on YearMonthIntervalType(0,1) with nullable=false, sortOrder=List('a DESC NULLS LAST)")
.excludeCH("sorting on YearMonthIntervalType(0,1) with nullable=false, sortOrder=List('a DESC NULLS FIRST)")
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
Comment on lines 2076 to +2077

@zml1206 zml1206 Jul 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated

enableSuite[GlutenStatisticsCollectionSuite]
// The output byte size of Velox is different
.includeCH("SPARK-33687: analyze all tables in a specific database")
Expand Down
4 changes: 3 additions & 1 deletion gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import org.apache.spark.sql.streaming._
// scalastyle:off line.size.limit

class VeloxTestSettings extends BackendTestSettings {
import SuiteSettings._
private val ansiNoFallback: Boolean =
sys.props.get(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key).contains("false")
enableSuite[GlutenStringFunctionsSuite]
Expand Down Expand Up @@ -1151,7 +1152,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenSQLQuerySuiteAE]
enableSuite[GlutenWindowQuerySuite]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
// TODO: 4.x enableSuite[GlutenSparkSessionExtensionSuite] // 1 failure
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenGroupBasedDeleteFromTableSuite]
enableSuite[GlutenDeltaBasedDeleteFromTableSuite]
enableSuite[GlutenDataFrameToSchemaSuite]
Expand Down
6 changes: 3 additions & 3 deletions gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ package org.apache.spark.sql

import org.apache.gluten.config.GlutenConfig

import org.apache.spark.sql.shim.GlutenTestsTrait

class GlutenSparkSessionExtensionSuite extends SparkSessionExtensionSuite with GlutenTestsTrait {
class GlutenSparkSessionExtensionSuite
extends SparkSessionExtensionSuite
with GlutenTestsCommonTrait {

testGluten("customColumnarOp") {
val extensions = DummyFilterColmnarHelper.create {
Expand Down
Loading