diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 94c164c5f45..d1dd0ffaa3a 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -121,164 +121,6 @@ Result set:: } } -opensearch.sql.query.analysis.enabled -===================================== - -Description ------------ - -You can disable query analyzer to bypass strict syntactic and semantic analysis. - -1. The default value is true. -2. This setting is node scope. -3. This setting can be updated dynamically. - - -Example -------- - -You can update the setting with a new value like this. - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_sql/settings -d '{ - "transient" : { - "opensearch.sql.query.analysis.enabled" : "false" - } - }' - -Result set:: - - { - "acknowledged" : true, - "persistent" : { }, - "transient" : { - "opensearch" : { - "sql" : { - "query" : { - "analysis" : { - "enabled" : "false" - } - } - } - } - } - } - -opensearch.sql.query.analysis.semantic.suggestion -================================================= - -Description ------------ - -You can enable query analyzer to suggest correct field names for quick fix. - -1. The default value is false. -2. This setting is node scope. -3. This setting can be updated dynamically. - - -Example 1 ---------- - -You can update the setting with a new value like this. - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_sql/settings -d '{ - "transient" : { - "opensearch.sql.query.analysis.semantic.suggestion" : "true" - } - }' - -Result set:: - - { - "acknowledged" : true, - "persistent" : { }, - "transient" : { - "opensearch" : { - "sql" : { - "query" : { - "analysis" : { - "semantic" : { - "suggestion" : "true" - } - } - } - } - } - } - } - -Example 2 ---------- - -Query result after the setting updated is like: - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X POST localhost:9200/_plugins/_sql -d '{ - "query" : "SELECT first FROM accounts" - }' - -Result set:: - - { - "error" : { - "reason" : "Invalid SQL query", - "details" : "Field [first] cannot be found or used here. Did you mean [firstname]?", - "type" : "SemanticAnalysisException" - }, - "status" : 400 - } - -opensearch.sql.query.analysis.semantic.threshold -================================================ - -Description ------------ - -Because query analysis needs to build semantic context in memory, index with large number of field would be skipped. You can update it to apply analysis to smaller or larger index as needed. - -1. The default value is 200. -2. This setting is node scope. -3. This setting can be updated dynamically. - - -Example -------- - -You can update the setting with a new value like this. - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_sql/settings -d '{ - "transient" : { - "opensearch.sql.query.analysis.semantic.threshold" : "50" - } - }' - -Result set:: - - { - "acknowledged" : true, - "persistent" : { }, - "transient" : { - "opensearch" : { - "sql" : { - "query" : { - "analysis" : { - "semantic" : { - "threshold" : "50" - } - } - } - } - } - } - } - opensearch.sql.query.response.format ==================================== diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java index bae91029fa5..38c22057afb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java @@ -39,9 +39,6 @@ import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_FETCH_SIZE; import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_KEEPALIVE; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_SLOWLOG; import static org.opensearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; @@ -87,35 +84,6 @@ public void slowLogSetting() { ); } - @Section(3) - public void queryAnalysisEnabledSetting() { - docSetting( - QUERY_ANALYSIS_ENABLED, - "You can disable query analyzer to bypass strict syntactic and semantic analysis.", - false - ); - } - - @Section(4) - public void semanticSuggestionSetting() { - docSetting( - QUERY_ANALYSIS_SEMANTIC_SUGGESTION, - "You can enable query analyzer to suggest correct field names for quick fix.", - true, - "SELECT first FROM accounts" - ); - } - - @Section(5) - public void semanticAnalysisThresholdSetting() { - docSetting( - QUERY_ANALYSIS_SEMANTIC_THRESHOLD, - "Because query analysis needs to build semantic context in memory, index with large number of field " + - "would be skipped. You can update it to apply analysis to smaller or larger index as needed.", - 50 - ); - } - @Section(6) public void responseFormatSetting() { docSetting( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java index 703483bb56b..c9e1a9f82c1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java @@ -59,11 +59,11 @@ public void noGroupKeySingleFuncOverAggWithoutAliasShouldPass() { @Test public void noGroupKeyMaxAddMinShouldPass() { JSONObject response = executeJdbcRequest(String.format( - "SELECT MAX(age) + MIN(age) as add " + + "SELECT MAX(age) + MIN(age) as addValue " + "FROM %s", Index.ACCOUNT.getName())); - verifySchema(response, schema("add", "add", "long")); + verifySchema(response, schema("MAX(age) + MIN(age)", "addValue", "long")); verifyDataRows(response, rows(60)); } @@ -111,14 +111,14 @@ public void hasGroupKeyAvgOnIntegerShouldPass() { @Test public void hasGroupKeyMaxAddMinShouldPass() { JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, MAX(age) + MIN(age) as add " + + "SELECT gender, MAX(age) + MIN(age) as addValue " + "FROM %s " + "GROUP BY gender", Index.ACCOUNT.getName())); verifySchema(response, schema("gender", null, "text"), - schema("add", "add", "long")); + schema("MAX(age) + MIN(age)", "addValue", "long")); verifyDataRows(response, rows("m", 60), rows("f", 60)); @@ -156,14 +156,14 @@ public void noGroupKeyLogMaxAddMinShouldPass() { @Test public void hasGroupKeyLogMaxAddMinShouldPass() { JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, Log(MAX(age) + MIN(age)) as log " + + "SELECT gender, Log(MAX(age) + MIN(age)) as logValue " + "FROM %s " + "GROUP BY gender", Index.ACCOUNT.getName())); verifySchema(response, schema("gender", null, "text"), - schema("log", "log", "double")); + schema("Log(MAX(age) + MIN(age))", "logValue", "double")); verifyDataRows(response, rows("m", 4.0943445622221d), rows("f", 4.0943445622221d)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java index f5f6f93bb45..4d1451f8c55 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java @@ -78,7 +78,7 @@ public void countTest() throws IOException { Assert.assertThat(getIntAggregationValue(result, "COUNT(*)", "value"), equalTo(1000)); } - @Test + @Ignore("The distinct is not supported in new engine") public void countDistinctTest() { JSONObject response = executeJdbcRequest( String.format("SELECT COUNT(distinct gender) FROM %s", TEST_INDEX_ACCOUNT)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java index acbce653324..d5eae6ec064 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java @@ -237,17 +237,16 @@ public void sqlPersistentOnlySettingTest() throws IOException { /** * Both transient and persistent settings are applied for same settings. - * This is similiar to _cluster/settings behavior + * This is similar to _cluster/settings behavior */ @Test public void sqlCombinedSettingTest() throws IOException { String settings = "{" + " \"transient\": {" + - " \"opensearch.sql.metrics.rollingwindow\": \"3700\"," + - " \"opensearch.sql.query.analysis.semantic.suggestion\" : \"false\"" + + " \"opensearch.sql.metrics.rollingwindow\": \"3700\"" + " }," + " \"persistent\": {" + - " \"opensearch.sql.query.analysis.semantic.suggestion\" : \"true\"" + + " \"opensearch.sql.query.slowlog\" : \"2\"" + " }" + "}"; JSONObject actual = updateViaSQLSettingsAPI(settings); @@ -257,11 +256,7 @@ public void sqlCombinedSettingTest() throws IOException { " \"opensearch\" : {" + " \"sql\" : {" + " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"true\"" + - " }" + - " }" + + " \"slowlog\" : \"2\"" + " }" + " }" + " }" + @@ -271,13 +266,6 @@ public void sqlCombinedSettingTest() throws IOException { " \"sql\" : {" + " \"metrics\" : {" + " \"rollingwindow\" : \"3700\"" + - " }," + - " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"false\"" + - " }" + - " }" + " }" + " }" + " }" + @@ -300,7 +288,7 @@ public void ignoreNonSQLSettingsTest() throws IOException { " \"search.max_keep_alive\": \"24h\"" + " }," + " \"persistent\": {" + - " \"opensearch.sql.query.analysis.semantic.suggestion\": \"true\"," + + " \"opensearch.sql.query.slowlog\": \"2\"," + " \"opensearch.alerting.metrics.rollingwindow\": \"3700\"," + " \"thread_pool.analyze.queue_size\": \"16\"" + " }" + @@ -312,11 +300,7 @@ public void ignoreNonSQLSettingsTest() throws IOException { " \"opensearch\" : {" + " \"sql\" : {" + " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"true\"" + - " }" + - " }" + + " \"slowlog\" : \"2\"" + " }" + " }" + " }" + diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java index 55095f7eadf..ee6f4da43e1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java @@ -32,9 +32,6 @@ import static org.opensearch.rest.RestStatus.BAD_REQUEST; import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; import java.io.IOException; import org.junit.Assert; @@ -46,7 +43,6 @@ import org.opensearch.sql.legacy.antlr.semantic.SemanticAnalysisException; import org.opensearch.sql.legacy.antlr.syntax.SyntaxAnalysisException; import org.opensearch.sql.legacy.exception.SqlFeatureNotImplementedException; -import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.utils.StringUtils; /** @@ -71,57 +67,6 @@ public void unsupportedOperatorShouldThrowSyntaxException() { ); } - @Test - public void unsupportedOperatorShouldSkipAnalysisAndThrowOtherExceptionIfAnalyzerDisabled() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_ENABLED, "false"), - () -> queryShouldThrowException( - "SELECT * FROM opensearch-sql_test_index_bank WHERE age <=> 1", - SqlParseException.class - ) - ); - } - - @Test - public void suggestionForWrongFieldNameShouldBeProvidedIfSuggestionEnabled() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_SUGGESTION, "true"), - () -> queryShouldThrowSemanticException( - "SELECT * FROM opensearch-sql_test_index_bank b WHERE a.balance = 1000", - "Field [a.balance] cannot be found or used here.", - "Did you mean [b.balance]?" - ) - ); - } - - @Test - public void wrongFieldNameShouldPassIfIndexMappingIsVeryLarge() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_THRESHOLD, "5"), - () -> queryShouldPassAnalysis( - "SELECT * FROM opensearch-sql_test_index_bank WHERE age123 = 1") - ); - } - - /* - @Test - public void useNewAddedFieldShouldPass() throws Exception { - // 1.Make sure new add fields not there originally - String query = "SELECT salary FROM opensearch-sql_test_index_bank WHERE education = 'PhD'"; - queryShouldThrowSemanticException(query, "Field [education] cannot be found or used here."); - - // 2.Index an document with fields not present in mapping previously - String docWithNewFields = "{\"account_number\":12345,\"education\":\"PhD\",\"salary\": \"10000\"}"; - IndexResponse resp = client().index(new IndexRequest().index("opensearch-sql_test_index_bank"). - source(docWithNewFields, JSON)).get(); - - Assert.assertEquals(RestStatus.CREATED, resp.status()); - - // 3.Same query should pass - executeQuery(query); - } - */ - @Test public void nonExistingFieldNameShouldThrowSemanticException() { queryShouldThrowSemanticException( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java index 327c5c6e5d7..c3e5029c536 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java @@ -284,34 +284,37 @@ public void castIntFieldToDoubleWithAliasOrderByTest() throws IOException { public void castIntFieldToFloatWithoutAliasJdbcFormatGroupByTest() { JSONObject response = executeJdbcRequest( "SELECT CAST(balance AS FLOAT) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY balance DESC LIMIT 5"); + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY balance ORDER BY balance DESC LIMIT 5"); verifySchema(response, schema("CAST(balance AS FLOAT)", null, "float")); verifyDataRows(response, - rows(22026), - rows(23285), - rows(36038), - rows(39063), - rows(45493)); + rows(49989.0), + rows(49795.0), + rows(49741.0), + rows(49671.0), + rows(49587.0)); } @Test public void castIntFieldToFloatWithAliasJdbcFormatGroupByTest() { JSONObject response = executeJdbcRequest( - "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + - "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY jdbc_float_alias ASC LIMIT 5"); + "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + + " FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " GROUP BY jdbc_float_alias " + + " ORDER BY jdbc_float_alias ASC " + + " LIMIT 5"); verifySchema(response, - schema("jdbc_float_alias", "jdbc_float_alias", "float")); + schema("CAST(balance AS FLOAT)", "jdbc_float_alias", "float")); verifyDataRows(response, - rows("22026.0"), - rows("23285.0"), - rows("36038.0"), - rows("39063.0"), - rows("45493.0")); + rows(1011.0), + rows(10116.0), + rows(10138.0), + rows(10147.0), + rows(10178.0)); } @Test @@ -631,7 +634,8 @@ public void operatorReplace() { ); } - @Test + @Ignore("The LOCATE function is not implemented in new SQL engine. https://github" + + ".com/opensearch-project/sql/issues/74") public void operatorLocate() throws IOException { String query = "SELECT LOCATE('a', lastname, 0) FROM " + TEST_INDEX_ACCOUNT + @@ -666,7 +670,8 @@ public void ltrim() throws IOException { ); } - @Test + @Ignore("The ASCII function is not implemented in new SQL engine. https://github" + + ".com/opensearch-project/sql/issues/73") public void ascii() throws IOException { assertThat( executeQuery("SELECT ASCII(lastname) FROM " + TEST_INDEX_ACCOUNT diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java index 60a49a51722..9a96de37b94 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java @@ -157,11 +157,11 @@ public void testSubtractLongWithLongReturnsLong() { */ @Test public void testDayOfWeekWithKeywordReturnsText() { - JSONObject response = executeJdbcRequest("SELECT DAY_OF_WEEK(insert_time) FROM " + JSONObject response = executeJdbcRequest("SELECT DAYOFWEEK(insert_time) FROM " + TestsConstants.TEST_INDEX_ONLINE + " LIMIT 2"); verifySchema(response, - schema("DAY_OF_WEEK(insert_time)", null, "integer")); + schema("DAYOFWEEK(insert_time)", null, "integer")); } @Test diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index 33b52b99a71..d69b95570a9 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -30,9 +30,6 @@ import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; import static org.opensearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; import com.alibaba.druid.sql.parser.ParserException; @@ -294,11 +291,7 @@ private boolean isCursorDisabled() { private static ColumnTypeProvider performAnalysis(String sql) { LocalClusterState clusterState = LocalClusterState.state(); - SqlAnalysisConfig config = new SqlAnalysisConfig( - clusterState.getSettingValue(QUERY_ANALYSIS_ENABLED), - clusterState.getSettingValue(QUERY_ANALYSIS_SEMANTIC_SUGGESTION), - clusterState.getSettingValue(QUERY_ANALYSIS_SEMANTIC_THRESHOLD) - ); + SqlAnalysisConfig config = new SqlAnalysisConfig(false, false, 200); OpenSearchLegacySqlAnalyzer analyzer = new OpenSearchLegacySqlAnalyzer(config); Optional outputColumnType = analyzer.analyze(sql, clusterState); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java index 7db1706aea8..068efd9ce81 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java @@ -51,9 +51,6 @@ public class SqlSettings { public static final String SQL_ENABLED = "opensearch.sql.enabled"; public static final String QUERY_SLOWLOG = "opensearch.sql.query.slowlog"; public static final String QUERY_RESPONSE_FORMAT = "opensearch.sql.query.response.format"; - public static final String QUERY_ANALYSIS_ENABLED = "opensearch.sql.query.analysis.enabled"; - public static final String QUERY_ANALYSIS_SEMANTIC_SUGGESTION = "opensearch.sql.query.analysis.semantic.suggestion"; - public static final String QUERY_ANALYSIS_SEMANTIC_THRESHOLD = "opensearch.sql.query.analysis.semantic.threshold"; public static final String METRICS_ROLLING_WINDOW = "opensearch.sql.metrics.rollingwindow"; public static final String METRICS_ROLLING_INTERVAL = "opensearch.sql.metrics.rollinginterval"; @@ -70,14 +67,6 @@ public SqlSettings() { settings.put(QUERY_RESPONSE_FORMAT, Setting.simpleString(QUERY_RESPONSE_FORMAT, Format.JDBC.getFormatName(), NodeScope, Dynamic)); - // Settings for new ANTLR query analyzer - settings.put(QUERY_ANALYSIS_ENABLED, Setting.boolSetting( - QUERY_ANALYSIS_ENABLED, true, NodeScope, Dynamic)); - settings.put(QUERY_ANALYSIS_SEMANTIC_SUGGESTION, Setting.boolSetting( - QUERY_ANALYSIS_SEMANTIC_SUGGESTION, false, NodeScope, Dynamic)); - settings.put(QUERY_ANALYSIS_SEMANTIC_THRESHOLD, Setting.intSetting( - QUERY_ANALYSIS_SEMANTIC_THRESHOLD, 200, NodeScope, Dynamic)); - settings.put(METRICS_ROLLING_WINDOW, Setting.longSetting(METRICS_ROLLING_WINDOW, 3600L, 2L, NodeScope, Dynamic)); settings.put(METRICS_ROLLING_INTERVAL, Setting.longSetting(METRICS_ROLLING_INTERVAL, 60L, 1L,