test: migrate regex extraction and position coverage to SQL fixtures

spark/src/test/resources/sql-tests/expressions/string/regexp_extract.sql

@@ -22,35 +22,54 @@ statement
 CREATE TABLE test_regexp_extract(s string) USING parquet
 
 statement
-INSERT INTO test_regexp_extract VALUES ('abc123def'), ('no match'), (NULL), ('xyz789'), ('hello world'), ('aa')
+INSERT INTO test_regexp_extract VALUES
+  ('abc123def'), ('no match'), (NULL), ('xyz789'), ('hello world'), ('aa'),
+  ('foo123bar'), ('abc'), ('def'), ('ab'), ('bb'), ('foobar'), ('foobaz'),
+  ('FOO123'), ('foo456'), ('bar789')
 
 -- group 0: entire match
 query expect_dispatch(regexp_extract)
-SELECT regexp_extract(s, r'\d+', 0) FROM test_regexp_extract
+SELECT s, regexp_extract(s, r'\d+', 0) FROM test_regexp_extract
 
 -- group 1: first capturing group
 query expect_dispatch(regexp_extract)
-SELECT regexp_extract(s, r'([a-z]+)(\d+)', 1) FROM test_regexp_extract
+SELECT s, regexp_extract(s, r'([a-z]+)(\d+)', 1) FROM test_regexp_extract
 
 -- group 2: second capturing group
 query expect_dispatch(regexp_extract)
-SELECT regexp_extract(s, r'([a-z]+)(\d+)', 2) FROM test_regexp_extract
+SELECT s, regexp_extract(s, r'([a-z]+)(\d+)', 2) FROM test_regexp_extract
 
 -- no match returns empty string
 query expect_dispatch(regexp_extract)
-SELECT regexp_extract(s, 'NOMATCH', 0) FROM test_regexp_extract
+SELECT s, regexp_extract(s, 'NOMATCH', 0) FROM test_regexp_extract
 
 -- backreference pattern (Java-only)
 query expect_dispatch(regexp_extract)
-SELECT regexp_extract(s, r'(\w)\1', 0) FROM test_regexp_extract
+SELECT s, regexp_extract(s, r'(\w)\1', 0) FROM test_regexp_extract
 
 -- lookahead (Java-only)
 query expect_dispatch(regexp_extract)
-SELECT regexp_extract(s, r'abc(?=\d)', 0) FROM test_regexp_extract
+SELECT s, regexp_extract(s, r'abc(?=\d)', 0) FROM test_regexp_extract
+
+query expect_dispatch(regexp_extract)
+SELECT s, regexp_extract(s, 'foo(?=bar)', 0) FROM test_regexp_extract
 
 -- embedded flags (Java-only)
 query expect_dispatch(regexp_extract)
-SELECT regexp_extract(s, '(?i)HELLO', 0) FROM test_regexp_extract
+SELECT s, regexp_extract(s, '(?i)HELLO', 0) FROM test_regexp_extract
+
+query expect_dispatch(regexp_extract)
+SELECT s, regexp_extract(s, r'(?i)(foo)(\d+)', 2) FROM test_regexp_extract
+
+-- Keep a separate all-null input batch, rather than only nulls mixed with matching rows.
+statement
+CREATE TABLE test_regexp_extract_nulls(s string) USING parquet
+
+statement
+INSERT INTO test_regexp_extract_nulls VALUES (NULL), (NULL), (NULL)
+
+query expect_dispatch(regexp_extract)
+SELECT regexp_extract(s, r'(\d+)', 1) FROM test_regexp_extract_nulls
 
 -- literal arguments
 query expect_dispatch(regexp_extract)

spark/src/test/resources/sql-tests/expressions/string/regexp_extract_all.sql

@@ -22,31 +22,51 @@ statement
 CREATE TABLE test_regexp_extract_all(s string) USING parquet
 
 statement
-INSERT INTO test_regexp_extract_all VALUES ('abc123def456'), ('no match'), (NULL), ('100-200-300'), ('hello world')
+INSERT INTO test_regexp_extract_all VALUES
+  ('abc123def456'), ('no match'), (NULL), ('100-200-300'), ('hello world'),
+  ('x1y2z3'), ('cat bat hat'), ('no vowels'), ('a1b2c3'), ('x9y8'), ('abc'), ('def'),
+  ('foobar foobaz fooqux'), ('abc123def'), ('xyz789')
 
 -- group 0: all entire matches
 query expect_dispatch(regexp_extract_all)
-SELECT regexp_extract_all(s, r'\d+', 0) FROM test_regexp_extract_all
+SELECT s, regexp_extract_all(s, r'\d+', 0) FROM test_regexp_extract_all
+
+query expect_dispatch(regexp_extract_all)
+SELECT s, regexp_extract_all(s, '[a-z]at', 0) FROM test_regexp_extract_all
 
 -- group 1: first capturing group from each match
 query expect_dispatch(regexp_extract_all)
-SELECT regexp_extract_all(s, r'([a-z]+)(\d+)', 1) FROM test_regexp_extract_all
+SELECT s, regexp_extract_all(s, r'([a-z]+)(\d+)', 1) FROM test_regexp_extract_all
+
+query expect_dispatch(regexp_extract_all)
+SELECT s, regexp_extract_all(s, r'(\d+)', 1) FROM test_regexp_extract_all
 
 -- group 2: second capturing group from each match
 query expect_dispatch(regexp_extract_all)
-SELECT regexp_extract_all(s, r'([a-z]+)(\d+)', 2) FROM test_regexp_extract_all
+SELECT s, regexp_extract_all(s, r'([a-z]+)(\d+)', 2) FROM test_regexp_extract_all
+
+-- Single-character groups exercise a capture from each adjacent match.
+query expect_dispatch(regexp_extract_all)
+SELECT s, regexp_extract_all(s, r'([a-z])(\d)', 1) FROM test_regexp_extract_all
+
+query expect_dispatch(regexp_extract_all)
+SELECT s, regexp_extract_all(s, r'([a-z])(\d)', 2) FROM test_regexp_extract_all
 
 -- no match returns empty array
 query expect_dispatch(regexp_extract_all)
-SELECT regexp_extract_all(s, 'NOMATCH', 0) FROM test_regexp_extract_all
+SELECT s, regexp_extract_all(s, 'NOMATCH', 0) FROM test_regexp_extract_all
 
 -- backreference pattern (Java-only)
 query expect_dispatch(regexp_extract_all)
-SELECT regexp_extract_all(s, r'(\d)\1', 0) FROM test_regexp_extract_all
+SELECT s, regexp_extract_all(s, r'(\d)\1', 0) FROM test_regexp_extract_all
+
+-- Multiple Java lookahead matches, followed by a non-match.
+query expect_dispatch(regexp_extract_all)
+SELECT s, regexp_extract_all(s, 'foo(?=ba[rz])', 0) FROM test_regexp_extract_all
 
 -- embedded flags (Java-only)
 query expect_dispatch(regexp_extract_all)
-SELECT regexp_extract_all(s, '(?i)[A-Z]+', 0) FROM test_regexp_extract_all
+SELECT s, regexp_extract_all(s, '(?i)[A-Z]+', 0) FROM test_regexp_extract_all
 
 -- literal arguments
 query expect_dispatch(regexp_extract_all)

spark/src/test/resources/sql-tests/expressions/string/regexp_extract_batches.sql

@@ -0,0 +1,32 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.batchSize=64
+
+-- One Parquet file keeps enough rows in one task to cross Arrow batch boundaries.
+statement
+CREATE TABLE test_regexp_extract_batches(id int, s string) USING parquet
+
+statement
+INSERT INTO test_regexp_extract_batches
+SELECT CAST(id AS INT),
+  CASE WHEN id % 7 = 0 THEN NULL ELSE concat('item_', CAST(id AS STRING), '_value') END
+FROM range(0, 5000, 1, 1)
+
+-- Keep the row ID so a shifted extraction or null bitmap cannot pass as the same result set.
+query expect_dispatch(regexp_extract)
+SELECT id, s, regexp_extract(s, r'item_(\d+)_value', 1) FROM test_regexp_extract_batches

spark/src/test/resources/sql-tests/expressions/string/regexp_instr.sql

@@ -22,27 +22,36 @@ statement
 CREATE TABLE test_regexp_instr(s string) USING parquet
 
 statement
-INSERT INTO test_regexp_instr VALUES ('abc123def'), ('no match'), (NULL), ('123xyz'), ('hello world'), ('aa')
+INSERT INTO test_regexp_instr VALUES
+  ('abc123def'), ('no match'), (NULL), ('123xyz'), ('hello world'), ('aa'),
+  ('456xyz'), ('abc123def456'), ('xyz'), ('abc'), ('def'), ('foobar'), ('foobaz'), ('xyz789')
 
 -- basic: position of first digit sequence
 query expect_dispatch(regexp_instr)
-SELECT regexp_instr(s, r'\d+', 0) FROM test_regexp_instr
+SELECT s, regexp_instr(s, r'\d+', 0) FROM test_regexp_instr
 
--- group 1 (still returns position of entire match per Spark semantics)
+-- Spark returns the position of the entire match for both third-argument values.
 query expect_dispatch(regexp_instr)
-SELECT regexp_instr(s, r'([a-z]+)(\d+)', 1) FROM test_regexp_instr
+SELECT s, regexp_instr(s, r'([a-z]+)(\d+)', 1) FROM test_regexp_instr
+
+query expect_dispatch(regexp_instr)
+SELECT s, regexp_instr(s, r'([a-z]+)(\d+)', 2) FROM test_regexp_instr
 
 -- no match returns 0
 query expect_dispatch(regexp_instr)
-SELECT regexp_instr(s, 'NOMATCH', 0) FROM test_regexp_instr
+SELECT s, regexp_instr(s, 'NOMATCH', 0) FROM test_regexp_instr
 
 -- backreference pattern (Java-only)
 query expect_dispatch(regexp_instr)
-SELECT regexp_instr(s, r'(\w)\1', 0) FROM test_regexp_instr
+SELECT s, regexp_instr(s, r'(\w)\1', 0) FROM test_regexp_instr
+
+-- lookahead (Java-only)
+query expect_dispatch(regexp_instr)
+SELECT s, regexp_instr(s, 'foo(?=bar)', 0) FROM test_regexp_instr
 
 -- embedded flags (Java-only)
 query expect_dispatch(regexp_instr)
-SELECT regexp_instr(s, '(?i)HELLO', 0) FROM test_regexp_instr
+SELECT s, regexp_instr(s, '(?i)HELLO', 0) FROM test_regexp_instr
 
 -- literal arguments
 query expect_dispatch(regexp_instr)

spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala

@@ -677,90 +677,6 @@ class CometRegExpJvmSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     }
   }
 
-  // ========== regexp_extract tests ==========
-
-  test("regexp_extract: basic group extraction") {
-    withSubjects("abc123def", "no match", null, "xyz789") {
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract(s, '([a-z]+)(\\\\d+)', 1) FROM t"))
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract(s, '([a-z]+)(\\\\d+)', 2) FROM t"))
-    }
-  }
-
-  test("regexp_extract: group 0 returns entire match") {
-    withSubjects("hello world", "foo123bar", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract(s, '\\\\d+', 0) FROM t"))
-    }
-  }
-
-  test("regexp_extract: no match returns empty string") {
-    withSubjects("abc", "def", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract(s, '\\\\d+', 0) FROM t"))
-    }
-  }
-
-  test("regexp_extract: backreference pattern (Java-only)") {
-    withSubjects("aa", "ab", "bb", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract(s, '(\\\\w)\\\\1', 0) FROM t"))
-    }
-  }
-
-  test("regexp_extract: lookahead pattern (Java-only)") {
-    withSubjects("foobar", "foobaz", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract(s, 'foo(?=bar)', 0) FROM t"))
-    }
-  }
-
-  test("regexp_extract: embedded flags (Java-only)") {
-    withSubjects("FOO123", "foo456", "bar789") {
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract(s, '(?i)(foo)(\\\\d+)', 2) FROM t"))
-    }
-  }
-
-  test("regexp_extract: all-null column") {
-    withSubjects(null, null, null) {
-      checkSparkAnswerAndOperator(sql("SELECT regexp_extract(s, '(\\\\d+)', 1) FROM t"))
-    }
-  }
-
-  // ========== regexp_extract_all tests ==========
-
-  test("regexp_extract_all: basic extraction of all matches") {
-    withSubjects("abc123def456", "no match", null, "x1y2z3") {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract_all(s, '(\\\\d+)', 1) FROM t"))
-    }
-  }
-
-  test("regexp_extract_all: group 0 returns full matches") {
-    withSubjects("cat bat hat", "no vowels", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract_all(s, '[a-z]at', 0) FROM t"))
-    }
-  }
-
-  test("regexp_extract_all: multiple groups") {
-    withSubjects("a1b2c3", "x9y8", null) {
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract_all(s, '([a-z])(\\\\d)', 1) FROM t"))
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract_all(s, '([a-z])(\\\\d)', 2) FROM t"))
-    }
-  }
-
-  test("regexp_extract_all: no matches returns empty array") {
-    withSubjects("abc", "def") {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract_all(s, '\\\\d+', 0) FROM t"))
-    }
-  }
-
-  test("regexp_extract_all: lookahead pattern (Java-only)") {
-    withSubjects("foobar foobaz fooqux") {
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract_all(s, 'foo(?=ba[rz])', 0) FROM t"))
-    }
-  }
-
   // ========== regexp_replace tests ==========
 
   test("regexp_replace: basic replacement") {
@@ -847,33 +763,6 @@ class CometRegExpJvmSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     }
   }
 
-  // ========== regexp_instr tests ==========
-
-  test("regexp_instr: basic position finding") {
-    withSubjects("abc123def", "no match", null, "456xyz") {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_instr(s, '\\\\d+', 0) FROM t"))
-    }
-  }
-
-  test("regexp_instr: specific group position") {
-    withSubjects("abc123def456", "xyz", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_instr(s, '([a-z]+)(\\\\d+)', 1) FROM t"))
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_instr(s, '([a-z]+)(\\\\d+)', 2) FROM t"))
-    }
-  }
-
-  test("regexp_instr: no match returns 0") {
-    withSubjects("abc", "def", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_instr(s, '\\\\d+', 0) FROM t"))
-    }
-  }
-
-  test("regexp_instr: lookahead (Java-only)") {
-    withSubjects("foobar", "foobaz", null) {
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_instr(s, 'foo(?=bar)', 0) FROM t"))
-    }
-  }
-
   // ========== split tests ==========
 
   test("split: basic regex split") {
@@ -912,19 +801,7 @@ class CometRegExpJvmSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     }
   }
 
-  // ========== multi-batch and combined tests ==========
-
-  test("regexp_extract: many rows spanning multiple batches") {
-    withTable("t") {
-      sql("CREATE TABLE t (s STRING) USING parquet")
-      val values = (0 until 5000)
-        .map(i => if (i % 7 == 0) "(NULL)" else s"('item_${i}_value')")
-        .mkString(", ")
-      sql(s"INSERT INTO t VALUES $values")
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract(s, 'item_(\\\\d+)_value', 1) FROM t"))
-    }
-  }
+  // ========== combined tests ==========
 
   test("all regexp expressions combined in one query") {
     withSubjects("abc123def456", "hello world", null, "aa") {
@@ -939,15 +816,4 @@ class CometRegExpJvmSuite extends CometTestBase with AdaptiveSparkPlanHelper {
           |""".stripMargin))
     }
   }
-
-  test("expressions with no native path always run via the JVM dispatcher") {
-    withSubjects("abc123def", "no match", null, "xyz789") {
-      // regexp_extract / regexp_extract_all / regexp_instr have no native rust path, so they
-      // always run on Comet via the JVM codegen dispatcher rather than falling back to Spark.
-      checkSparkAnswerAndOperator(
-        sql("SELECT s, regexp_extract(s, '([a-z]+)(\\\\d+)', 2) FROM t"))
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_extract_all(s, '\\\\d+', 0) FROM t"))
-      checkSparkAnswerAndOperator(sql("SELECT s, regexp_instr(s, '\\\\d+', 0) FROM t"))
-    }
-  }
 }