diff --git a/integ-test/build.gradle b/integ-test/build.gradle index dd3053473b6..9f51ba2c222 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1239,6 +1239,52 @@ task integTestRemote(type: RestIntegTestTask) { // === Excludes: CalcitePPLBasicIT route divergence === // REGEXP filter throws a backend NullPointerException on the AE route. excludeTestsMatching '*CalcitePPLBasicIT.testRegexpFilter' + + // === Excludes: percentile is approximate on AE (DataFusion) but exact on v2/Calcite === + excludeTestsMatching '*StatsCommandIT.testStatsPercentileWithNull' + excludeTestsMatching '*StatsCommandIT.testStatsPercentileByNullValue' + excludeTestsMatching '*StatsCommandIT.testStatsPercentileByNullValueNonNullBucket' + excludeTestsMatching '*StatsCommandIT.testStatsPercentileBySpan' + excludeTestsMatching '*CalcitePPLAggregationIT.testPercentile' + excludeTestsMatching '*CalcitePPLAggregationIT.testPercentileShortcutsFloatingPoint' + + // === Excludes: span() time-field bucketing differs on the AE route === + excludeTestsMatching '*StatsCommandIT.testStatsBySpanTimeWithNullBucket' + excludeTestsMatching '*CalciteChartCommandIT.testChartMaxValueByTimestampSpanDayAndWeek' + + // === Excludes: float/half_float arithmetic keeps 32-bit precision on AE === + excludeTestsMatching '*CalcitePPLBuiltinFunctionIT.testDivide' + excludeTestsMatching '*CalcitePPLBuiltinFunctionIT.testModFloatAndNegative' + excludeTestsMatching '*CalcitePPLBuiltinFunctionIT.testModShouldReturnWiderTypes' + + // === Excludes: date_format/strftime render some tokens differently on AE === + excludeTestsMatching '*DateTimeFunctionIT.testDateFormat' + excludeTestsMatching '*CalciteDateTimeFunctionIT.testStrftimeWithDateFields' + + // === Excludes: unix_timestamp drops sub-second precision on AE === + excludeTestsMatching '*DateTimeFunctionIT.testUnixTimestampWithTimestampString' + + // === Excludes: json_set/json_delete with a $-prefixed path is a no-op on AE === + excludeTestsMatching '*CalcitePPLJsonBuiltinFunctionIT.testJsonSetWithDollarPrefixedPath' + excludeTestsMatching '*CalcitePPLJsonBuiltinFunctionIT.testJsonDeleteWithDollarPrefixedPath' + + // === Excludes: dedup surviving-row selection is non-deterministic on AE === + excludeTestsMatching '*CalcitePPLDedupIT.testDedupComplex' + excludeTestsMatching '*CalcitePPLDedupIT.testDedupExpr' + excludeTestsMatching '*CalcitePPLDedupIT.testConsecutiveImplicitFallbackV2' + + // === Excludes: same-index union conflates on AE (delegated predicate leak) === + excludeTestsMatching '*CalciteUnionCommandIT.testUnionThreeSubsearches' + excludeTestsMatching '*CalciteUnionCommandIT.testUnionMidPipeline_SingleExplicitDataset' + + // === Excludes: rename * returns columns in a different order on AE === + excludeTestsMatching '*CalcitePPLRenameIT.testRenameFullWildcardExcludesMetadataFields' + + // === Excludes: otel_logs multi-value field can't load into the parquet store === + excludeTestsMatching '*CalciteChartCommandIT.testChartLimit0WithUseOther' + excludeTestsMatching '*CalciteChartCommandIT.testChartLimitTopWithUseOther' + excludeTestsMatching '*CalciteChartCommandIT.testChartLimitBottomWithUseOther' + excludeTestsMatching '*CalciteChartCommandIT.testChartLimitTopWithMinAgg' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java index 36dcf5697c8..3805759c65e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java @@ -92,29 +92,24 @@ public void testTimestampRootColumnSpaceFormat() throws IOException { verifyDataRows(result, rows("2024-03-15 10:30:00")); } - /** - * DATE-mapped col: AE widens to TIMESTAMP at scan time; value must use space separator, not ISO - * {@code T}. - */ + /** DATE-mapped col: AE preserves the DATE type (date UDT) and renders {@code yyyy-MM-dd}. */ @Test public void testDateRootColumnYmdFormat() throws IOException { String query = "source=" + INDEX + " | where d = '2024-03-15' | fields d"; assertRoutedToAnalyticsEngine(query); JSONObject result = executeQuery(query); - verifySchema(result, schema("d", "timestamp")); - verifyDataRows(result, rows("2024-03-15 00:00:00")); + verifySchema(result, schema("d", "date")); + verifyDataRows(result, rows("2024-03-15")); } - /** TIME-mapped col: AE widens to TIMESTAMP; value must use space separator, not ISO {@code T}. */ + /** TIME-mapped col: AE preserves the TIME type (time UDT) and renders {@code HH:mm:ss}. */ @Test public void testTimeRootColumnHmsFormat() throws IOException { String query = "source=" + INDEX + " | sort t | head 1 | fields t"; assertRoutedToAnalyticsEngine(query); JSONObject result = executeQuery(query); - verifySchema(result, schema("t", "timestamp")); - Assert.assertFalse( - "Time-mapped column must not surface as ISO T-separator literal", - result.getJSONArray("datarows").getJSONArray(0).getString(0).contains("T")); + verifySchema(result, schema("t", "time")); + verifyDataRows(result, rows("10:30:00")); } /** Eval-derived TIMESTAMP follows the same wire-format contract as a root column. */ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index e687751ef0c..ed360bc79d4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -9,6 +9,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; +import static org.opensearch.sql.util.Capability.BIN_TIME_FIELD_BUCKETING; +import static org.opensearch.sql.util.Capability.MULTI_VALUE_FIELD_LOAD; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -20,6 +22,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteChartCommandIT extends PPLIntegTestCase { @Override @@ -28,7 +31,13 @@ public void init() throws Exception { enableCalcite(); loadIndex(Index.BANK); loadIndex(Index.BANK_WITH_NULL_VALUES); - loadIndex(Index.OTELLOGS); + // otel_logs has a multi-value array for a scalar-mapped field, which the parquet store rejects + // at bulk load (MULTI_VALUE_FIELD_LOAD); skip the load on the AE route so it doesn't abort + // init() for the otel-independent tests. The otel tests themselves are + // @RequiresCapability-gated. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.OTELLOGS); + } loadIndex(Index.TIME_TEST_DATA); loadIndex(Index.EVENTS_NULL); } @@ -142,6 +151,11 @@ public void testChartMaxValueOverCategoryByTimestampSpanWeek() throws IOExceptio } @Test + @RequiresCapability( + value = BIN_TIME_FIELD_BUCKETING, + note = + "span=2weeks anchors the bucket to a different week origin on the AE route" + + " (BIN_TIME_FIELD_BUCKETING).") public void testChartMaxValueByTimestampSpanDayAndWeek() throws IOException { JSONObject result = executeQuery( @@ -165,6 +179,11 @@ public void testChartMaxValueByTimestampSpanDayAndWeek() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimit0WithUseOther() throws IOException { JSONObject result = executeQuery( @@ -208,6 +227,11 @@ public void testChartLimit0WithUseOther() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimitTopWithUseOther() throws IOException { JSONObject result = executeQuery( @@ -230,6 +254,11 @@ public void testChartLimitTopWithUseOther() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimitBottomWithUseOther() throws IOException { JSONObject result = executeQuery( @@ -246,6 +275,11 @@ public void testChartLimitBottomWithUseOther() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimitTopWithMinAgg() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java index ef0c0599b57..cb74251f5d8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +18,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ppl.DateTimeFunctionIT; +import org.opensearch.sql.util.RequiresCapability; public class CalciteDateTimeFunctionIT extends DateTimeFunctionIT { @Override @@ -73,6 +75,9 @@ public void testStrftimeWithVariousInputTypes() throws IOException { } @Test + @RequiresCapability( + value = DATETIME_FORMAT_RENDERING, + note = "strftime renders sub-second precision differently on the AE route.") public void testStrftimeWithDateFields() throws IOException { // Test strftime with different date field types from indices loadIndex(Index.DATE_FORMATS); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index a5937d06f31..a2ab93b6599 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TELEMETRY; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -28,6 +29,7 @@ import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLAggregationIT extends PPLIntegTestCase { @@ -967,6 +969,9 @@ public void testTake() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testPercentile() throws IOException { JSONObject actual = executeQuery( @@ -1184,6 +1189,9 @@ public void testPercentileShortcutsWithDecimals() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testPercentileShortcutsFloatingPoint() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java index 58f4cb849b3..cbd94683fd1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NULL_MISSING; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; +import static org.opensearch.sql.util.Capability.FLOAT_ARITHMETIC_PRECISION; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -21,6 +22,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -255,6 +257,9 @@ public void testModWithSortAndFields() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float modulo keeps 32-bit precision on AE; v2 widens to double.") public void testModFloatAndNegative() throws IOException { JSONObject actual = executeQuery( @@ -267,6 +272,9 @@ public void testModFloatAndNegative() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float modulo keeps 32-bit precision on AE; v2 widens to double.") public void testModShouldReturnWiderTypes() throws IOException { JSONObject actual = executeQuery( @@ -347,6 +355,9 @@ public void testSignAndRound() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float/half_float division keeps 32-bit precision on AE; v2 widens to double.") public void testDivide() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java index 71e9e69e3ae..9c93b12e6ac 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java @@ -7,12 +7,14 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DUPLICATION_NULLABLE; +import static org.opensearch.sql.util.Capability.DEDUP_NONDETERMINISTIC; import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLDedupIT extends PPLIntegTestCase { @@ -95,6 +97,9 @@ public void testDedupMultipleFieldsKeepEmpty() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = "dedup CONSECUTIVE behavior diverges on the AE route.") public void testConsecutiveImplicitFallbackV2() throws IOException { JSONObject actual = executeQuery( @@ -252,6 +257,10 @@ public void testReorderDedupFieldsShouldNotAffectResult() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = + "dedup surviving-duplicate selection diverges on the AE route (no stable merge order).") public void testDedupComplex() throws IOException { JSONObject actual = executeQuery(String.format("source=%s | dedup 1 name", TEST_INDEX_DUPLICATION_NULLABLE)); @@ -364,6 +373,10 @@ public void testSortThenDedupKeepEmpty() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = + "dedup surviving-duplicate selection diverges on the AE route (no stable merge order).") public void testDedupExpr() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java index 99af10302ae..02ebae95e8e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.JSON_DOLLAR_PATH; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -15,6 +16,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLJsonBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -30,7 +32,12 @@ public void init() throws Exception { loadIndex(Index.PEOPLE2); loadIndex(Index.BANK); loadIndex(Index.JSON_TEST); - loadIndex(Index.GAME_OF_THRONES); + // game_of_thrones has a multi-value array for the scalar-mapped `titles` field, which the + // parquet store rejects at bulk load; skip it on the AE route so it doesn't abort init() for + // the rest of the suite. No test in this class queries game_of_thrones. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.GAME_OF_THRONES); + } } @Test @@ -297,6 +304,9 @@ public void testJsonSetPartialSet() throws IOException { } @Test + @RequiresCapability( + value = JSON_DOLLAR_PATH, + note = "json_set with a $-prefixed path is a no-op on the AE route (JSON_DOLLAR_PATH).") public void testJsonSetWithDollarPrefixedPath() throws IOException { // Issue #5167: json_set with $.key path should not double-prefix JSONObject actual = @@ -313,6 +323,9 @@ public void testJsonSetWithDollarPrefixedPath() throws IOException { } @Test + @RequiresCapability( + value = JSON_DOLLAR_PATH, + note = "json_delete with a $-prefixed path is a no-op on the AE route (JSON_DOLLAR_PATH).") public void testJsonDeleteWithDollarPrefixedPath() throws IOException { // Issue #5167: json_delete with $.key path should remove the key JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 3503d7c533c..6bac3032bd2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; +import static org.opensearch.sql.util.Capability.WILDCARD_COLUMN_ORDER; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -21,6 +22,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLRenameIT extends PPLIntegTestCase { @@ -204,6 +206,9 @@ public void testRenameFullWildcard() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = "rename * returns columns in a different order on the AE route.") public void testRenameFullWildcardExcludesMetadataFields() throws IOException { JSONObject result = executeQuery(String.format("source = %s | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java index 1dbd34357ab..c0151a3b232 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOCATIONS_TYPE_CONFLICT; +import static org.opensearch.sql.util.Capability.SAME_INDEX_UNION_CONFLATION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -18,6 +19,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteUnionCommandIT extends PPLIntegTestCase { @@ -48,6 +50,9 @@ public void testBasicUnionTwoSubsearches() throws IOException { } @Test + @RequiresCapability( + value = SAME_INDEX_UNION_CONFLATION, + note = "same-index union conflates on the AE route (delegated predicate leak).") public void testUnionThreeSubsearches() throws IOException { JSONObject result = executeQuery( @@ -155,6 +160,9 @@ public void testUnionAllDatasetsDifferentSchemas() throws IOException { } @Test + @RequiresCapability( + value = SAME_INDEX_UNION_CONFLATION, + note = "same-index union conflates on the AE route (delegated predicate leak).") public void testUnionMidPipeline_SingleExplicitDataset() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 1a2911794f9..97b1d9601a2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -7,7 +7,9 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; import static org.opensearch.sql.util.Capability.DOC_MUTATION; +import static org.opensearch.sql.util.Capability.UNIX_TIMESTAMP_SUBSECOND; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -1219,6 +1221,9 @@ void verifyDateFormat(String date, String type, String format, String formatted) } @Test + @RequiresCapability( + value = DATETIME_FORMAT_RENDERING, + note = "date_format renders some tokens differently on the AE route.") public void testDateFormat() throws IOException { String timestamp = "1998-01-31 13:14:15.012345"; String timestampFormat = @@ -1369,6 +1374,11 @@ public void testUnixTimeStamp() throws IOException { } @Test + @RequiresCapability( + value = UNIX_TIMESTAMP_SUBSECOND, + note = + "unix_timestamp drops the sub-second fraction on the AE route" + + " (UNIX_TIMESTAMP_SUBSECOND).") public void testUnixTimestampWithTimestampString() throws IOException { var result = executeQuery( @@ -1557,7 +1567,14 @@ public void testTimestampDiff() throws IOException { "source=%s | eval f = timestampdiff(YEAR, '1997-01-01 00:00:00', '2001-03-06" + " 00:00:00') | fields f", TEST_INDEX_DATE)); - verifySchema(result, schema("f", null, isCalciteEnabled() ? "bigint" : "timestamp")); + // The AE route runs the Calcite path, returning bigint even though the cluster's calcite + // setting reads false. + verifySchema( + result, + schema( + "f", + null, + isCalciteEnabled() || isAnalyticsParquetIndicesEnabled() ? "bigint" : "timestamp")); verifySome(result.getJSONArray("datarows"), rows(4)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index adbe04bfd8a..7417fd112ec 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -9,6 +9,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATE_NULL; +import static org.opensearch.sql.util.Capability.BIN_TIME_FIELD_BUCKETING; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -19,6 +21,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.util.RequiresCapability; public class StatsCommandIT extends PPLIntegTestCase { @@ -622,8 +625,10 @@ public void testStatsPercentileWithMin() throws IOException { "source=%s | eval decimal=ceil(balance/100000.0) | stats percentile(decimal, 50)," + " min(decimal)", TEST_INDEX_BANK)); + // The AE route runs the Calcite path, so it returns the Calcite (double) type even though the + // cluster's calcite setting reads false; treat it like the Calcite branch. String returnType = "bigint"; - if (isCalciteEnabled()) { + if (isCalciteEnabled() || isAnalyticsParquetIndicesEnabled()) { returnType = "double"; } @@ -635,6 +640,9 @@ public void testStatsPercentileWithMin() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileWithNull() throws IOException { JSONObject response = executeQuery( @@ -673,6 +681,9 @@ public void testStatsPercentileWhere() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileByNullValue() throws IOException { JSONObject response = executeQuery( @@ -691,6 +702,9 @@ public void testStatsPercentileByNullValue() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileByNullValueNonNullBucket() throws IOException { JSONObject response = executeQuery( @@ -708,6 +722,9 @@ public void testStatsPercentileByNullValueNonNullBucket() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileBySpan() throws IOException { JSONObject response = executeQuery( @@ -755,6 +772,9 @@ public void testDisableLegacyPreferred() throws IOException { } @Test + @RequiresCapability( + value = BIN_TIME_FIELD_BUCKETING, + note = "span() time bucketing differs on the AE route (bucket set/null bucket).") public void testStatsBySpanTimeWithNullBucket() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index b0e119bffb1..276ab01da4c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -9,12 +9,14 @@ import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.SCALED_FLOAT_TYPE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class SystemFunctionIT extends PPLIntegTestCase { @@ -53,6 +55,9 @@ public void typeof_sql_types() throws IOException { } @Test + @RequiresCapability( + value = SCALED_FLOAT_TYPE, + note = "typeof(scaled_float) is bigint on the AE route, not double.") public void typeof_opensearch_types() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index d54b3ac74a9..b7e95bfc4f1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -279,7 +279,92 @@ public enum Capability { */ ADDTOTALS_JOIN_PANIC( "addtotals crashes the DataFusion backend with a join panic (out-of-range slice index) on the" - + " analytics-engine route."); + + " analytics-engine route."), + + /** + * {@code percentile}/{@code median} is approximate on the analytics-engine route (DataFusion's + * approx percentile) but exact on the v2/Calcite path, so percentile values, null-bucket rows, + * and by-span groupings diverge. + */ + PERCENTILE_APPROXIMATE( + "percentile/median is approximate on the analytics-engine route (DataFusion) but exact on the" + + " v2/Calcite path, so the values diverge."), + + /** + * Arithmetic over {@code float}/{@code half_float}-typed fields keeps 32-bit float precision on + * the analytics-engine route (DataFusion), whereas the v2/Calcite path widens to double, so the + * least-significant digits diverge (e.g. 0.2 vs 0.19999981). + */ + FLOAT_ARITHMETIC_PRECISION( + "Arithmetic over float/half_float fields keeps 32-bit precision on the analytics-engine route" + + " (DataFusion) but widens to double on the v2/Calcite path, so the values diverge in" + + " the least-significant digits."), + + /** + * Datetime formatting functions ({@code date_format}, {@code strftime}) render some tokens / + * sub-second precision differently on the analytics-engine route than on the v2/Calcite path. + */ + DATETIME_FORMAT_RENDERING( + "date_format/strftime render some format tokens and sub-second precision differently on the" + + " analytics-engine route than the v2/Calcite path."), + + /** + * {@code json_set}/{@code json_delete} with a {@code $}-prefixed path ({@code $.key}) is a no-op + * on the analytics-engine route (the JSON UDF doesn't strip the {@code $} prefix), whereas the + * v2/Calcite path applies the modification. + */ + JSON_DOLLAR_PATH( + "json_set/json_delete with a $-prefixed path is a no-op on the analytics-engine route (the" + + " JSON UDF doesn't handle the $ prefix), whereas the v2/Calcite path applies it."), + + /** + * A dataset whose document has a multi-value array for a scalar-mapped field can't be bulk-loaded + * into the parquet/composite store ({@code Cannot accept multiple values for field ...}), so + * tests reading that dataset fail at setup on the analytics-engine route. + */ + MULTI_VALUE_FIELD_LOAD( + "A multi-value array for a scalar-mapped field can't be bulk-loaded into the parquet store on" + + " the analytics-engine route, so the dataset fails to load."), + + /** + * {@code dedup} returns a different/non-deterministic row set on the analytics-engine route — the + * engine merges per-fragment batches without a stable tiebreaker, so which duplicate survives + * (and {@code CONSECUTIVE=true} behavior) diverges from the v2/Calcite path. + */ + DEDUP_NONDETERMINISTIC( + "dedup returns a different row set on the analytics-engine route: per-fragment merge order" + + " has no stable tiebreaker, so the surviving duplicate (and CONSECUTIVE behavior)" + + " diverges."), + + /** + * {@code union}/{@code multisearch} over subsearches that read the same index conflates on the + * analytics-engine route: a delegated predicate from one branch leaks onto the co-located shard + * fragment and is applied to all branches, so counts/rows are wrong. Same root cause as {@link + * #MULTISEARCH_SAME_INDEX_CONFLATION} / {@link #APPENDPIPE_MAIN_RESULT_DROPPED}. + */ + SAME_INDEX_UNION_CONFLATION( + "union over same-index subsearches conflates on the analytics-engine route: a delegated" + + " predicate from one branch leaks across the co-located shard fragment, so counts/rows" + + " are wrong."), + + /** + * A wildcard projection/rename ({@code rename * as ...}, {@code fields *}) returns columns in a + * different order on the analytics-engine route (e.g. not mapping order) than the v2/Calcite + * path, so row-position-sensitive assertions diverge even though the values are correct. + */ + WILDCARD_COLUMN_ORDER( + "A wildcard projection/rename returns columns in a different order on the analytics-engine" + + " route than the v2/Calcite path."), + + /** + * {@code unix_timestamp()} over a timestamp string with sub-second precision drops the fractional + * seconds on the analytics-engine route (e.g. {@code unix_timestamp('1984-06-06 + * 12:00:00.123456')} returns {@code 455371200} instead of {@code 455371200.123456}), whereas the + * v2/Calcite path preserves them. + */ + UNIX_TIMESTAMP_SUBSECOND( + "unix_timestamp() drops sub-second precision on the analytics-engine route (returns whole" + + " seconds), whereas the v2/Calcite path preserves the fractional seconds."); private final String reason;