From 8dd09046530a85d7df8f614296de11b55eb3c899 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 16 Jun 2021 14:50:46 -0400 Subject: [PATCH 01/20] Add sql integTest script for opensearch Signed-off-by: Peter Zhu --- integtest.sh | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 integtest.sh diff --git a/integtest.sh b/integtest.sh new file mode 100755 index 00000000000..8be8831a60f --- /dev/null +++ b/integtest.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +set -e + +function usage() { + echo "" + echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster." + echo "--------------------------------------------------------------------------" + echo "Usage: $0 [args]" + echo "" + echo "Required arguments:" + echo "None" + echo "" + echo "Optional arguments:" + echo -e "-b BIND_ADDRESS\t, defaults to localhost | 127.0.0.1, can be changed to any IP or domain name for the cluster location." + echo -e "-p BIND_PORT\t, defaults to 9200 or 5601 depends on OpenSearch or Dashboards, can be changed to any port for the cluster location." + echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." + echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." + echo -e "-h\tPrint this message." + echo "--------------------------------------------------------------------------" +} + +while getopts ":hb:p:s:c:" arg; do + case $arg in + h) + usage + exit 1 + ;; + b) + BIND_ADDRESS=$OPTARG + ;; + p) + BIND_PORT=$OPTARG + ;; + s) + SECURITY_ENABLED=$OPTARG + ;; + c) + CREDENTIAL=$OPTARG + ;; + :) + echo "-${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}" + exit 1 + ;; + esac +done + + +if [ -z "$BIND_ADDRESS" ] +then + BIND_ADDRESS="localhost" +fi + +if [ -z "$BIND_PORT" ] +then + BIND_PORT="9200" +fi + +if [ -z "$SECURITY_ENABLED" ] +then + SECURITY_ENABLED="true" +fi + +if [ -z "$CREDENTIAL" ] +then + CREDENTIAL="admin:admin" + USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` + PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` +fi + +./gradlew integTest -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain + From 5a6a3e859c14a85ce9be839f95a670b18cbfe64f Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Wed, 30 Jun 2021 13:42:19 -0700 Subject: [PATCH 02/20] Refactor readme (#148) Signed-off-by: Chen Dai --- README.md | 193 +++++++----------------------------------------------- 1 file changed, 22 insertions(+), 171 deletions(-) diff --git a/README.md b/README.md index bf764154594..0a663254249 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,22 @@ [![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/sql/) ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) -# OpenSearch SQL + +- [OpenSearch SQL](#opensearch-sql) +- [Highlights](#highlights) +- [Documentation](#documentation) +- [Contributing](#contributing) +- [Attribution](#attribution) +- [Code of Conduct](#code-of-conduct) +- [Security](#security) +- [License](#license) +- [Copyright](#copyright) -OpenSearch enables you to extract insights out of OpenSearch using the familiar SQL query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you. +# OpenSearch SQL -## SQL Related Projects +OpenSearch enables you to extract insights out of OpenSearch using the familiar SQL or Piped Processing Language (PPL) query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you. The following projects have been merged into this repository as separate folders as of July 9, 2020. Please refer to links below for details. This document will focus on the SQL plugin for OpenSearch. @@ -21,197 +30,39 @@ The following projects have been merged into this repository as separate folders * [Query Workbench](https://github.com/opensearch-project/sql/tree/main/workbench) -## Documentation - -Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://docs-beta.opensearch.org/) for detailed information on installing and configuring plugin. Looking to contribute? Read the instructions on [Developer Guide](./DEVELOPER_GUIDE.rst) and then submit a patch! - -## SQL Engine V2 - -Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported the new released Piped Processing Language. However, it was experimental and disabled by default for SQL query processing. With most important features and full testing complete, now we're ready to promote it as our default SQL query engine. Please find more details in [SQL Engine V2 - Release Notes](/docs/dev/NewSQLEngine.md). - - -## Setup - -Install as plugin: build plugin from source code by following the instruction in Build section and install it to your OpenSearch. - -After doing this, you need to restart the OpenSearch server. Otherwise you may get errors like `Invalid index name [sql], must not start with '']; ","status":400}`. - - -## Build - -The package uses the [Gradle](https://docs.gradle.org/4.10.2/userguide/userguide.html) build system. - -1. Checkout this package from version control. -2. To build from command line set `JAVA_HOME` to point to a JDK >=14 -3. Run `./gradlew build` - - -## Basic Usage - -To use the feature, send requests to the `_plugins/_sql` URI. You can use a request parameter or the request body (recommended). Note that for backward compatibility, old `_opendistro/_sql` endpoint is still available, though any future API will be only accessible by new OpenSearch endpoint. - -* Simple query - -``` -POST https://:/_plugins/_sql -{ - "query": "SELECT * FROM my-index LIMIT 50" -} -``` - -* Explain SQL to OpenSearch query DSL -``` -POST _plugins/_sql/_explain -{ - "query": "SELECT * FROM my-index LIMIT 50" -} -``` - -* For a sample curl command with the OpenSearch Security plugin, try: -``` -curl -XPOST https://localhost:9200/_plugins/_sql -u admin:admin -k -d '{"query": "SELECT * FROM my-index LIMIT 10"}' -H 'Content-Type: application/json' -``` - - -## SQL Usage - -* Query - - SELECT * FROM bank WHERE age >30 AND gender = 'm' +## Highlights -* Aggregation +Besides basic filtering and aggregation, OpenSearch SQL also supports complex queries, such as querying semi-structured data, JOINs, set operations, sub-queries etc. Beyond the standard functions, OpenSearch functions are provided for better analytics and visualization. Please check our [documentation](#documentation) for more details. - SELECT COUNT(*),SUM(age),MIN(age) as m, MAX(age),AVG(age) - FROM bank - GROUP BY gender - HAVING m >= 20 - ORDER BY SUM(age), m DESC +Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported both SQL and Piped Processing Language. Please find more details in [SQL Engine V2 - Release Notes](/docs/dev/NewSQLEngine.md). -* Join - SELECT b1.firstname, b1.lastname, b2.age - FROM bank b1 - LEFT JOIN bank b2 - ON b1.age = b2.age AND b1.state = b2.state - -* Show - - SHOW TABLES LIKE ban% - DESCRIBE TABLES LIKE bank - -* Delete - - DELETE FROM bank WHERE age >30 AND gender = 'm' - - -## Beyond SQL - -* Search - - SELECT address FROM bank WHERE address = matchQuery('880 Holmes Lane') ORDER BY _score DESC LIMIT 3 - -* Nested Field - - + - - SELECT address FROM bank b, b.nestedField e WHERE b.state = 'WA' and e.name = 'test' - - + - SELECT address, nested(nestedField.name) - FROM bank - WHERE nested(nestedField, nestedField.state = 'WA' AND nestedField.name = 'test') - OR nested(nestedField.state) = 'CA' - -* Aggregations - - + range age group 20-25,25-30,30-35,35-40 - - SELECT COUNT(age) FROM bank GROUP BY range(age, 20,25,30,35,40) - - + range date group by day - - SELECT online FROM online GROUP BY date_histogram(field='insert_time','interval'='1d') - - + range date group by your config - - SELECT online FROM online GROUP BY date_range(field='insert_time','format'='yyyy-MM-dd' ,'2014-08-18','2014-08-17','now-8d','now-7d','now-6d','now') - -* OpenSearch Geographic - - SELECT * FROM locations WHERE GEO_BOUNDING_BOX(fieldname,100.0,1.0,101,0.0) - -* Select type or pattern - - SELECT * FROM indexName/type - SELECT * FROM index* - - -## SQL Features +## Documentation -* SQL Select -* SQL Delete -* SQL Where -* SQL Order By -* SQL Group By -* SQL Having -* SQL Inner Join -* SQL Left Join -* SQL Show -* SQL Describe -* SQL AND & OR -* SQL Like -* SQL COUNT distinct -* SQL In -* SQL Between -* SQL Aliases -* SQL Not Null -* SQL(OpenSearch) Date -* SQL avg() -* SQL count() -* SQL max() -* SQL min() -* SQL sum() -* SQL Nulls -* SQL isnull() -* SQL floor -* SQL trim -* SQL log -* SQL log10 -* SQL substring -* SQL round -* SQL sqrt -* SQL concat_ws -* SQL union and minus +Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://docs-beta.opensearch.org/) for detailed information on installing and configuring plugin. -## JDBC Support -Please check out JDBC driver repository for more details. +## Contributing -## Beyond SQL features +See [developer guide](DEVELOPER_GUIDE.rst) and [how to contribute to this project](CONTRIBUTING.md). -* OpenSearch TopHits -* OpenSearch MISSING -* OpenSearch STATS -* OpenSearch GEO_INTERSECTS -* OpenSearch GEO_BOUNDING_BOX -* OpenSearch GEO_DISTANCE -* OpenSearch GEOHASH_GRID aggregation ## Attribution This project is based on the Apache 2.0-licensed [elasticsearch-sql](https://github.com/NLPchina/elasticsearch-sql) project. Thank you [eliranmoyal](https://github.com/eliranmoyal), [shi-yuan](https://github.com/shi-yuan), [ansjsun](https://github.com/ansjsun) and everyone else who contributed great code to that project. Read this for more details [Attributions](./docs/attributions.md). + ## Code of Conduct This project has adopted an [Open Source Code of Conduct](./CODE_OF_CONDUCT.md). -## Security issue notifications +## Security If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. -## Licensing +## License See the [LICENSE](./LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. From 47f5534944622d07b5f7e980f6a841e5b9e0034b Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Wed, 30 Jun 2021 16:02:21 -0700 Subject: [PATCH 03/20] Build against OpenSearch 1.0.0 and bump artifact version to 1.0.0.0 (#146) * Bump OpenSearch version from rc1 to 1.0.0 Signed-off-by: Chen Dai * Rename JDBC artifact by removing -rc1 Signed-off-by: Chen Dai * Remove rc1 qualifier in build workflow Signed-off-by: Chen Dai * Remove rc1 from build tools version Signed-off-by: Chen Dai * Fix IT failure Signed-off-by: Chen Dai * Rollback build tools to rc1 due to known issue Signed-off-by: Chen Dai * Bump CLI version Signed-off-by: Chen Dai * Bump query workbench version Signed-off-by: Chen Dai * Build against 1.0.0 Signed-off-by: Chen Dai * Update release notes drafter Signed-off-by: Chen Dai * Update nodejs to 10.24.1 Signed-off-by: Chen Dai --- .github/workflows/draft-release-notes-workflow.yml | 4 ++-- .github/workflows/sql-test-and-build-workflow.yml | 2 +- .github/workflows/sql-workbench-release-workflow.yml | 4 ++-- .../workflows/sql-workbench-test-and-build-workflow.yml | 4 ++-- build.gradle | 4 ++-- sql-cli/src/opensearch_sql_cli/__init__.py | 2 +- sql-jdbc/build.gradle | 2 +- workbench/opensearch_dashboards.json | 4 ++-- workbench/package.json | 8 ++++---- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 5afb4ff55e3..cc68eeab4a0 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -3,7 +3,7 @@ name: Release Drafter on: push: branches: - - develop + - main jobs: update_release_draft: @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 1.0.0.0-rc1 + version: 1.0.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 8ad294d8538..816056bd33d 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -25,7 +25,7 @@ jobs: - name: Build OpenSearch working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false + run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false - name: Build with Gradle run: ./gradlew build assemble diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index b22f7649b33..0e8e7126903 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -8,7 +8,7 @@ on: env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: '1.0' - OPENSEARCH_PLUGIN_VERSION: 1.0.0.0-rc1 + OPENSEARCH_PLUGIN_VERSION: 1.0.0.0 jobs: @@ -38,7 +38,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v1 with: - node-version: '10.23.1' + node-version: '10.24.1' - name: Move Workbench to Plugins Dir run: | diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 05a6541e341..cad24e68998 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: '1.0' - OPENSEARCH_PLUGIN_VERSION: 1.0.0.0-rc1 + OPENSEARCH_PLUGIN_VERSION: 1.0.0.0 jobs: @@ -27,7 +27,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v1 with: - node-version: '10.23.1' + node-version: '10.24.1' - name: Move Workbench to Plugins Dir run: | diff --git a/build.gradle b/build.gradle index 1b69cf810ba..85523b88591 100644 --- a/build.gradle +++ b/build.gradle @@ -26,7 +26,7 @@ buildscript { ext { - opensearch_version = "1.0.0-rc1" + opensearch_version = "1.0.0" } repositories { @@ -60,7 +60,7 @@ ext { } allprojects { - version = "${opensearchVersion}.0-rc1" + version = "${opensearchVersion}.0" plugins.withId('java') { sourceCompatibility = targetCompatibility = "1.8" diff --git a/sql-cli/src/opensearch_sql_cli/__init__.py b/sql-cli/src/opensearch_sql_cli/__init__.py index 2ffe4957253..e9f870deba7 100644 --- a/sql-cli/src/opensearch_sql_cli/__init__.py +++ b/sql-cli/src/opensearch_sql_cli/__init__.py @@ -22,4 +22,4 @@ express or implied. See the License for the specific language governing permissions and limitations under the License. """ -__version__ = "1.0.0.0-rc1" +__version__ = "1.0.0.0" diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 955a75e3666..548b34a9be2 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -43,7 +43,7 @@ plugins { group 'org.opensearch.client' // keep version in sync with version in Driver source -version '1.0.0.0-rc1' +version '1.0.0.0' boolean snapshot = "true".equals(System.getProperty("build.snapshot", "false")); if (snapshot) { diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index a873b67a9fc..0ee32f9b98a 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "queryWorkbenchDashboards", - "version": "1.0.0.0-rc1", - "opensearchDashboardsVersion": "1.0.0-rc1", + "version": "1.0.0.0", + "opensearchDashboardsVersion": "1.0.0", "server": true, "ui": true, "requiredPlugins": ["navigation"], diff --git a/workbench/package.json b/workbench/package.json index 54d009704ec..3e78f6a26b2 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -1,13 +1,13 @@ { "name": "opensearch-query-workbench", - "version": "1.0.0.0-rc1", + "version": "1.0.0.0", "description": "Query Workbench", "main": "index.js", "license": "Apache-2.0", "homepage": "https://github.com/opensearch-project/sql/tree/main/workbench", "opensearchDashboards": { - "version": "1.0.0-rc1", - "templateVersion": "1.0.0-rc1" + "version": "1.0.0", + "templateVersion": "1.0.0" }, "repository": { "type": "git", @@ -47,7 +47,7 @@ "tslint-plugin-prettier": "^2.0.1" }, "engines": { - "node": "10.23.1", + "node": "10.24.1", "yarn": "^1.21.1" }, "resolutions": { From 399881d49b52a873e1a8ddac19bacc3c7901b446 Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Thu, 1 Jul 2021 09:01:21 -0700 Subject: [PATCH 04/20] Change grammar and add UT (#150) Signed-off-by: Chen Dai --- sql/src/main/antlr/OpenSearchSQLParser.g4 | 2 +- .../org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 18c75b94ffc..fe5526621f3 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -72,7 +72,7 @@ adminStatement ; showStatement - : SHOW TABLES tableFilter? + : SHOW TABLES tableFilter ; describeStatement diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 53de19a0fd0..cf1ca36f028 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -160,4 +160,9 @@ public void canParseOrderByClause() { "SELECT name, age FROM test ORDER BY name ASC NULLS FIRST, age DESC NULLS LAST")); } + @Test + public void canNotParseShowStatementWithoutFilterClause() { + assertThrows(SyntaxCheckException.class, () -> parser.parse("SHOW TABLES")); + } + } From c37c4239096020e39d5904a23a6496b02ceed91c Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Thu, 1 Jul 2021 09:37:42 -0700 Subject: [PATCH 05/20] Add release notes for OpenSearch GA (#151) * Add release notes Signed-off-by: Chen Dai * Change release date Signed-off-by: Chen Dai * Add bug fixes section Signed-off-by: Chen Dai --- .../opensearch-sql.release-notes-1.0.0.0.md | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-1.0.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-1.0.0.0.md b/release-notes/opensearch-sql.release-notes-1.0.0.0.md new file mode 100644 index 00000000000..9f4ba692783 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-1.0.0.0.md @@ -0,0 +1,54 @@ +## 2021-07-12 Version 1.0.0.0 + +Compatible with OpenSearch and OpenSearch Dashboards Version 1.0.0 + +### Enhancements + +* Support querying a data stream ([#56](https://github.com/opensearch-project/sql/pull/56)) + +### Bug Fixes + +* Bug Fix: Enable legacy settings in new setting action ([#97](https://github.com/opensearch-project/sql/pull/97)) +* Fix NPE for SHOW statement without filter ([#150](https://github.com/opensearch-project/sql/pull/150)) + +### OpenSearch Migration + +* Remove debug logging in ODBC driver ([#27](https://github.com/opensearch-project/sql/pull/27)) +* Update workbench nav category to opensearch ([#28](https://github.com/opensearch-project/sql/pull/28)) +* fix opendistro related renaming for sql-cli ([#29](https://github.com/opensearch-project/sql/pull/29)) +* Fix issue of workbench not outputting errors ([#32](https://github.com/opensearch-project/sql/pull/32)) +* Update issue template with multiple labels ([#34](https://github.com/opensearch-project/sql/pull/34)) +* SQL/PPL and JDBC package renaming ([#54](https://github.com/opensearch-project/sql/pull/54)) +* Upgrade dependencies to address high severity CVE-2021-20270 ([#61](https://github.com/opensearch-project/sql/pull/61)) +* ODBC folder, file and code renaming ([#62](https://github.com/opensearch-project/sql/pull/62)) +* Update workbench documentation links, complete renaming ([#67](https://github.com/opensearch-project/sql/pull/67)) +* Update sqli-cli documentation links to OpenSearch ([#72](https://github.com/opensearch-project/sql/pull/72)) +* Remove opensearch.sql.engine.new.enabled setting ([#70](https://github.com/opensearch-project/sql/pull/70)) +* SQL/PPL API endpoint backward compatibility ([#66](https://github.com/opensearch-project/sql/pull/66)) +* Remove opensearch.sql.query.analysis.* related settings ([#76](https://github.com/opensearch-project/sql/pull/76)) +* Remove opensearch.sql.query.response.format setting ([#77](https://github.com/opensearch-project/sql/pull/77)) +* Migrate #1097: Adding support to NOT REGEXP_QUERY ([#79](https://github.com/opensearch-project/sql/pull/79)) +* Migrate #1083: Support long literals in SQL/PPL ([#80](https://github.com/opensearch-project/sql/pull/80)) +* Change strategy to test connectivity between ODBC driver and SQL plugin ([#69](https://github.com/opensearch-project/sql/pull/69)) +* Remove cursor enabling and fetch size setting ([#75](https://github.com/opensearch-project/sql/pull/75)) +* Disable DELETE clause by defaut and add opensearch.sql.delete.enabled setting ([#81](https://github.com/opensearch-project/sql/pull/81)) +* Support Plugin Settings Backwards Compatibility ([#82](https://github.com/opensearch-project/sql/pull/82)) +* Updated icon and background images in ODBC installers ([#84](https://github.com/opensearch-project/sql/pull/84)) +* Build SQL/PPL against OpenSearch rc1 and rename artifacts ([#83](https://github.com/opensearch-project/sql/pull/83)) +* Support text functions ASCII, LEFT, LOCATE, REPLACE in new engine ([#88](https://github.com/opensearch-project/sql/pull/88)) +* Update PowerBI custom connector .mez file for ODBC driver ([#90](https://github.com/opensearch-project/sql/pull/90)) +* Rename remaining beta1 references in sql-cli/workbench ([#91](https://github.com/opensearch-project/sql/pull/91)) +* Build SQL/PPL against OpenSearch 1.0 branch ([#94](https://github.com/opensearch-project/sql/pull/94)) +* Bump OpenSearch Dashboards version to 1.0 in Workbench ([#98](https://github.com/opensearch-project/sql/pull/98)) +* Add Integtest.sh for OpenSearch integtest setups ([#128](https://github.com/opensearch-project/sql/pull/128)) +* Merge develop into main ([#142](https://github.com/opensearch-project/sql/pull/142)) +* Build against OpenSearch 1.0.0 and bump artifact version to 1.0.0.0 ([#146](https://github.com/opensearch-project/sql/pull/146)) + +### Documentation + +* Migrate SQL/PPL, JDBC, ODBC docs to OpenSearch ([#68](https://github.com/opensearch-project/sql/pull/68)) +* Level up README markdown ([#148](https://github.com/opensearch-project/sql/pull/148)) + +### Infrastructure + +* Bump glob-parent from 5.1.1 to 5.1.2 in /workbench ([#125](https://github.com/opensearch-project/sql/pull/125)) From 4ca0eaba800c8fbee800b867a526df4ea35854b8 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 9 Jul 2021 17:37:49 -0400 Subject: [PATCH 06/20] Add sql dashboards tests for workbench Signed-off-by: Peter Zhu --- integtest.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/integtest.sh b/integtest.sh index 8be8831a60f..f5822fb49ac 100755 --- a/integtest.sh +++ b/integtest.sh @@ -16,11 +16,12 @@ function usage() { echo -e "-p BIND_PORT\t, defaults to 9200 or 5601 depends on OpenSearch or Dashboards, can be changed to any port for the cluster location." echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." + echo -e "-d DIRECTORY\tSome Repo has more than 1 directory / component, use this to give that directory for separate tests." echo -e "-h\tPrint this message." echo "--------------------------------------------------------------------------" } -while getopts ":hb:p:s:c:" arg; do +while getopts ":hb:p:s:c:d:" arg; do case $arg in h) usage @@ -38,6 +39,9 @@ while getopts ":hb:p:s:c:" arg; do c) CREDENTIAL=$OPTARG ;; + d) + DIRECTORY=$OPTARG + ;; :) echo "-${OPTARG} requires an argument" usage @@ -73,5 +77,23 @@ then PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` fi -./gradlew integTest -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain +if [ -z "$DIRECTORY" ] +then + DIRECTORY="root" +fi + +if [ "$DIRECTORY" = "workbench" ] +then + mv -v workbench ../ + cd ../ + rm -rf sql + cd workbench + yarn osd bootstrap + curl -s https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/accounts.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/accounts/_bulk?pretty' --data-binary @- > /dev/null 2>&1 + curl -s https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/employee_nested.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/employee_nested/_bulk?pretty' --data-binary @- > /dev/null 2>&1 + npx cypress run +else + ./gradlew integTest -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain + +fi From e2714e7e199c02074f1b3b99707afe347e4925a3 Mon Sep 17 00:00:00 2001 From: Lyndon Bauto <58273576+lyndonb-bq@users.noreply.github.com> Date: Fri, 23 Jul 2021 14:51:22 -0700 Subject: [PATCH 07/20] [1] Fixed aws init and shutdown behaviour (#163) --- .../src/sqlodbc/opensearch_communication.cpp | 42 +++++++++++++++++-- .../src/sqlodbc/opensearch_communication.h | 1 - 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/sql-odbc/src/sqlodbc/opensearch_communication.cpp b/sql-odbc/src/sqlodbc/opensearch_communication.cpp index 66ca0e03a6a..8fdfcae4f26 100644 --- a/sql-odbc/src/sqlodbc/opensearch_communication.cpp +++ b/sql-odbc/src/sqlodbc/opensearch_communication.cpp @@ -32,6 +32,8 @@ // clang-format off #include "opensearch_odbc.h" #include "mylog.h" +#include +#include #include #include #include @@ -121,6 +123,40 @@ static const std::string ERROR_RESPONSE_SCHEMA = R"EOF( } )EOF"; +namespace { + /** + * A helper class to initialize/shutdown AWS API once per DLL load/unload. + */ + class AwsSdkHelper { + public: + AwsSdkHelper() : + m_reference_count(0) { + } + + AwsSdkHelper& operator++() { + if (1 == ++m_reference_count) { + std::scoped_lock lock(m_mutex); + Aws::InitAPI(m_sdk_options); + } + return *this; + } + + AwsSdkHelper& operator--() { + if (0 == --m_reference_count) { + std::scoped_lock lock(m_mutex); + Aws::ShutdownAPI(m_sdk_options); + } + return *this; + } + + Aws::SDKOptions m_sdk_options; + std::atomic m_reference_count; + std::mutex m_mutex; + }; + + AwsSdkHelper AWS_SDK_HELPER; +} + void OpenSearchCommunication::AwsHttpResponseToString( std::shared_ptr< Aws::Http::HttpResponse > response, std::string& output) { // This function has some unconventional stream operations because we need @@ -237,13 +273,11 @@ OpenSearchCommunication::OpenSearchCommunication() #pragma clang diagnostic pop #endif // __APPLE__ { - LogMsg(OPENSEARCH_ALL, "Initializing Aws API."); - Aws::InitAPI(m_options); + ++AWS_SDK_HELPER; } OpenSearchCommunication::~OpenSearchCommunication() { - LogMsg(OPENSEARCH_ALL, "Shutting down Aws API."); - Aws::ShutdownAPI(m_options); + --AWS_SDK_HELPER; } std::string OpenSearchCommunication::GetErrorMessage() { diff --git a/sql-odbc/src/sqlodbc/opensearch_communication.h b/sql-odbc/src/sqlodbc/opensearch_communication.h index 6c141bfe086..4a328ece3b2 100644 --- a/sql-odbc/src/sqlodbc/opensearch_communication.h +++ b/sql-odbc/src/sqlodbc/opensearch_communication.h @@ -116,7 +116,6 @@ class OpenSearchCommunication { OpenSearchResultQueue m_result_queue; runtime_options m_rt_opts; std::string m_client_encoding; - Aws::SDKOptions m_options; std::string m_response_str; std::shared_ptr< Aws::Http::HttpClient > m_http_client; std::string m_error_message_to_user; From bcdd3f5af24fc3954e12c01de6787c8f52dee531 Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Wed, 28 Jul 2021 16:36:56 -0700 Subject: [PATCH 08/20] Support implicit type conversion from string to boolean (#166) * Support implicit type conversion for bool and string Signed-off-by: Chen Dai * Fix lucene query pushdown issue Signed-off-by: Chen Dai * Refactor lucene query methods Signed-off-by: Chen Dai * Refactor builtin repo methods Signed-off-by: Chen Dai * Add comparison test Signed-off-by: Chen Dai * Fix comparison test Signed-off-by: Chen Dai * Add doc test for user manual Signed-off-by: Chen Dai * Fix doc test Signed-off-by: Chen Dai * Fix design doc link Signed-off-by: Chen Dai * Fix RST render issue Signed-off-by: Chen Dai * Fix cast function pushdown issue Signed-off-by: Chen Dai * Improve javadoc for PR Signed-off-by: Chen Dai * Upload design doc Signed-off-by: Chen Dai * Add more user manual Signed-off-by: Chen Dai * Add more user manual Signed-off-by: Chen Dai * Fix doctest Signed-off-by: Chen Dai --- .../opensearch/sql/ast/expression/Cast.java | 26 ++- .../sql/data/type/ExprCoreType.java | 26 ++- .../opensearch/sql/data/type/ExprType.java | 10 + .../org/opensearch/sql/expression/DSL.java | 10 + .../function/BuiltinFunctionName.java | 2 + .../function/BuiltinFunctionRepository.java | 68 ++++++- .../expression/function/FunctionResolver.java | 8 +- .../operator/convert/TypeCastOperator.java | 26 +++ .../sql/data/type/ExprTypeTest.java | 13 ++ .../BuiltinFunctionRepositoryTest.java | 136 +++++++++++++- .../function/FunctionResolverTest.java | 4 +- .../function/WideningTypeRuleTest.java | 3 + .../convert/TypeCastOperatorTest.java | 54 ++++++ docs/dev/TypeConversion.md | 172 ++++++++++++++++++ docs/dev/img/type-hierarchy-tree-old.png | Bin 0 -> 27195 bytes ...type-hierarchy-tree-with-implicit-cast.png | Bin 0 -> 28295 bytes docs/user/general/datatypes.rst | 114 +++++++++++- .../correctness/expressions/cast.txt | 4 + .../data/type/OpenSearchDataType.java | 17 +- .../data/type/OpenSearchDataTypeTest.java | 6 + 20 files changed, 676 insertions(+), 23 deletions(-) create mode 100644 docs/dev/TypeConversion.md create mode 100644 docs/dev/img/type-hierarchy-tree-old.png create mode 100644 docs/dev/img/type-hierarchy-tree-with-implicit-cast.png diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 382ef325ff3..bd57d0a8a68 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -29,11 +29,13 @@ package org.opensearch.sql.ast.expression; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_BOOLEAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_BYTE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DOUBLE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_TIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_TIMESTAMP; @@ -49,6 +51,7 @@ import lombok.ToString; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.Node; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.FunctionName; /** @@ -60,9 +63,11 @@ @ToString public class Cast extends UnresolvedExpression { - private static Map CONVERTED_TYPE_FUNCTION_NAME_MAP = + private static final Map CONVERTED_TYPE_FUNCTION_NAME_MAP = new ImmutableMap.Builder() .put("string", CAST_TO_STRING.getName()) + .put("byte", CAST_TO_BYTE.getName()) + .put("short", CAST_TO_SHORT.getName()) .put("int", CAST_TO_INT.getName()) .put("integer", CAST_TO_INT.getName()) .put("long", CAST_TO_LONG.getName()) @@ -84,6 +89,25 @@ public class Cast extends UnresolvedExpression { */ private final UnresolvedExpression convertedType; + /** + * Check if the given function name is a cast function or not. + * @param name function name + * @return true if cast function, otherwise false. + */ + public static boolean isCastFunction(FunctionName name) { + return CONVERTED_TYPE_FUNCTION_NAME_MAP.containsValue(name); + } + + /** + * Get the cast function name for a given target data type. + * @param targetType target data type + * @return cast function name corresponding + */ + public static FunctionName getCastFunctionName(ExprType targetType) { + String type = targetType.typeName().toLowerCase(Locale.ROOT); + return CONVERTED_TYPE_FUNCTION_NAME_MAP.get(type); + } + /** * Get the converted type. * diff --git a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java index 3b37cfbf31a..92da09490cd 100644 --- a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java @@ -28,12 +28,13 @@ package org.opensearch.sql.data.type; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -62,16 +63,15 @@ public enum ExprCoreType implements ExprType { FLOAT(LONG), DOUBLE(FLOAT), - /** - * Boolean. - */ - BOOLEAN(UNDEFINED), - /** * String. */ STRING(UNDEFINED), + /** + * Boolean. + */ + BOOLEAN(STRING), /** * Date. @@ -108,6 +108,16 @@ public enum ExprCoreType implements ExprType { .put(STRING, "keyword") .build(); + private static final Set NUMBER_TYPES = + new ImmutableSet.Builder() + .add(BYTE) + .add(SHORT) + .add(INTEGER) + .add(LONG) + .add(FLOAT) + .add(DOUBLE) + .build(); + ExprCoreType(ExprCoreType... compatibleTypes) { for (ExprCoreType subType : compatibleTypes) { subType.parents.add(this); @@ -139,7 +149,7 @@ public static List coreTypes() { .collect(Collectors.toList()); } - public static List numberTypes() { - return ImmutableList.of(INTEGER, LONG, FLOAT, DOUBLE); + public static Set numberTypes() { + return NUMBER_TYPES; } } diff --git a/core/src/main/java/org/opensearch/sql/data/type/ExprType.java b/core/src/main/java/org/opensearch/sql/data/type/ExprType.java index a26f758b201..97c46ca4e59 100644 --- a/core/src/main/java/org/opensearch/sql/data/type/ExprType.java +++ b/core/src/main/java/org/opensearch/sql/data/type/ExprType.java @@ -58,6 +58,16 @@ default boolean isCompatible(ExprType other) { } } + /** + * Should cast this type to other type or not. By default, cast is always required + * if the given type is different from this type. + * @param other other data type + * @return true if cast is required, otherwise false + */ + default boolean shouldCast(ExprType other) { + return !this.equals(other); + } + /** * Get the parent type. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 560414592cd..42a49db2eef 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -592,6 +592,16 @@ public FunctionExpression castString(Expression value) { .compile(BuiltinFunctionName.CAST_TO_STRING.getName(), Arrays.asList(value)); } + public FunctionExpression castByte(Expression value) { + return (FunctionExpression) repository + .compile(BuiltinFunctionName.CAST_TO_BYTE.getName(), Arrays.asList(value)); + } + + public FunctionExpression castShort(Expression value) { + return (FunctionExpression) repository + .compile(BuiltinFunctionName.CAST_TO_SHORT.getName(), Arrays.asList(value)); + } + public FunctionExpression castInt(Expression value) { return (FunctionExpression) repository .compile(BuiltinFunctionName.CAST_TO_INT.getName(), Arrays.asList(value)); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 24e65d4b5d5..0f6feeb94ac 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -177,6 +177,8 @@ public enum BuiltinFunctionName { * Data Type Convert Function. */ CAST_TO_STRING(FunctionName.of("cast_to_string")), + CAST_TO_BYTE(FunctionName.of("cast_to_byte")), + CAST_TO_SHORT(FunctionName.of("cast_to_short")), CAST_TO_INT(FunctionName.of("cast_to_int")), CAST_TO_LONG(FunctionName.of("cast_to_long")), CAST_TO_FLOAT(FunctionName.of("cast_to_float")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index ebb432d7f03..3898af66828 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -11,10 +11,19 @@ package org.opensearch.sql.expression.function; +import static org.opensearch.sql.ast.expression.Cast.getCastFunctionName; +import static org.opensearch.sql.ast.expression.Cast.isCastFunction; + +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.Expression; @@ -47,15 +56,70 @@ public FunctionImplementation compile(FunctionName functionName, List resolvedSignature = + functionResolverMap.get(functionName).resolve(functionSignature); + + List sourceTypes = functionSignature.getParamTypeList(); + List targetTypes = resolvedSignature.getKey().getParamTypeList(); + FunctionBuilder funcBuilder = resolvedSignature.getValue(); + if (isCastFunction(functionName) || sourceTypes.equals(targetTypes)) { + return funcBuilder; + } + return castArguments(sourceTypes, targetTypes, funcBuilder); } else { throw new ExpressionEvaluationException( String.format("unsupported function name: %s", functionName.getFunctionName())); } } + + /** + * Wrap resolved function builder's arguments by cast function to cast input expression value + * to value of target type at runtime. For example, suppose unresolved signature is + * equal(BOOL,STRING) and its resolved function builder is F with signature equal(BOOL,BOOL). + * In this case, wrap F and return equal(BOOL, cast_to_bool(STRING)). + */ + private FunctionBuilder castArguments(List sourceTypes, + List targetTypes, + FunctionBuilder funcBuilder) { + return arguments -> { + List argsCasted = new ArrayList<>(); + for (int i = 0; i < arguments.size(); i++) { + Expression arg = arguments.get(i); + ExprType sourceType = sourceTypes.get(i); + ExprType targetType = targetTypes.get(i); + + if (isCastRequired(sourceType, targetType)) { + argsCasted.add(cast(arg, targetType)); + } else { + argsCasted.add(arg); + } + } + return funcBuilder.apply(argsCasted); + }; + } + + private boolean isCastRequired(ExprType sourceType, ExprType targetType) { + // TODO: Remove this special case after fixing all failed UTs + if (ExprCoreType.numberTypes().contains(sourceType) + && ExprCoreType.numberTypes().contains(targetType)) { + return false; + } + return sourceType.shouldCast(targetType); + } + + private Expression cast(Expression arg, ExprType targetType) { + FunctionName castFunctionName = getCastFunctionName(targetType); + if (castFunctionName == null) { + throw new ExpressionEvaluationException(StringUtils.format( + "Type conversion to type %s is not supported", targetType)); + } + return (Expression) compile(castFunctionName, ImmutableList.of(arg)); + } + } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/FunctionResolver.java b/core/src/main/java/org/opensearch/sql/expression/function/FunctionResolver.java index d9d01be891b..5bd63015a58 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/FunctionResolver.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/FunctionResolver.java @@ -20,6 +20,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Singular; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.exception.ExpressionEvaluationException; /** @@ -41,8 +42,10 @@ public class FunctionResolver { * If the {@link FunctionBuilder} exactly match the input {@link FunctionSignature}, return it. * If applying the widening rule, found the most match one, return it. * If nothing found, throw {@link ExpressionEvaluationException} + * + * @return function signature and its builder */ - public FunctionBuilder resolve(FunctionSignature unresolvedSignature) { + public Pair resolve(FunctionSignature unresolvedSignature) { PriorityQueue> functionMatchQueue = new PriorityQueue<>( Map.Entry.comparingByKey()); @@ -59,7 +62,8 @@ public FunctionBuilder resolve(FunctionSignature unresolvedSignature) { unresolvedSignature.formatTypes() )); } else { - return functionBundle.get(bestMatchEntry.getValue()); + FunctionSignature resolvedSignature = bestMatchEntry.getValue(); + return Pair.of(resolvedSignature, functionBundle.get(resolvedSignature)); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java index df6b6f935f7..5f94eb63ee8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java @@ -48,11 +48,13 @@ import java.util.stream.Stream; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprByteValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprShortValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; @@ -68,6 +70,8 @@ public class TypeCastOperator { */ public static void register(BuiltinFunctionRepository repository) { repository.register(castToString()); + repository.register(castToByte()); + repository.register(castToShort()); repository.register(castToInt()); repository.register(castToLong()); repository.register(castToFloat()); @@ -92,6 +96,28 @@ private static FunctionResolver castToString() { ); } + private static FunctionResolver castToByte() { + return FunctionDSL.define(BuiltinFunctionName.CAST_TO_BYTE.getName(), + impl(nullMissingHandling( + (v) -> new ExprByteValue(Short.valueOf(v.stringValue()))), BYTE, STRING), + impl(nullMissingHandling( + (v) -> new ExprByteValue(v.shortValue())), BYTE, DOUBLE), + impl(nullMissingHandling( + (v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN) + ); + } + + private static FunctionResolver castToShort() { + return FunctionDSL.define(BuiltinFunctionName.CAST_TO_SHORT.getName(), + impl(nullMissingHandling( + (v) -> new ExprShortValue(Short.valueOf(v.stringValue()))), SHORT, STRING), + impl(nullMissingHandling( + (v) -> new ExprShortValue(v.shortValue())), SHORT, DOUBLE), + impl(nullMissingHandling( + (v) -> new ExprShortValue(v.booleanValue() ? 1 : 0)), SHORT, BOOLEAN) + ); + } + private static FunctionResolver castToInt() { return FunctionDSL.define(BuiltinFunctionName.CAST_TO_INT.getName(), impl(nullMissingHandling( diff --git a/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java b/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java index 0dc8b8f4cf5..9beb11eb07a 100644 --- a/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java +++ b/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java @@ -33,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -58,6 +59,11 @@ public void isCompatible() { assertTrue(FLOAT.isCompatible(LONG)); assertTrue(FLOAT.isCompatible(INTEGER)); assertTrue(FLOAT.isCompatible(SHORT)); + assertTrue(BOOLEAN.isCompatible(STRING)); + } + + @Test + public void isNotCompatible() { assertFalse(INTEGER.isCompatible(DOUBLE)); assertFalse(STRING.isCompatible(DOUBLE)); assertFalse(INTEGER.isCompatible(UNKNOWN)); @@ -69,6 +75,13 @@ public void isCompatibleWithUndefined() { ExprCoreType.coreTypes().forEach(type -> assertFalse(UNDEFINED.isCompatible(type))); } + @Test + public void shouldCast() { + assertTrue(UNDEFINED.shouldCast(STRING)); + assertTrue(STRING.shouldCast(BOOLEAN)); + assertFalse(STRING.shouldCast(STRING)); + } + @Test public void getParent() { assertThat(((ExprType) () -> "test").getParent(), Matchers.contains(UNKNOWN)); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java index 6f8b3600ea9..d6b372a12a1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java @@ -29,20 +29,39 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.BYTE; +import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_BOOLEAN; +import com.google.common.collect.ImmutableList; import java.util.Arrays; +import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.env.Environment; @ExtendWith(MockitoExtension.class) @@ -62,6 +81,13 @@ class BuiltinFunctionRepositoryTest { @Mock private Environment emptyEnv; + private BuiltinFunctionRepository repo; + + @BeforeEach + void setUp() { + repo = new BuiltinFunctionRepository(mockMap); + } + @Test void register() { BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); @@ -73,8 +99,11 @@ void register() { @Test void compile() { + when(mockExpression.type()).thenReturn(UNDEFINED); + when(functionSignature.getParamTypeList()).thenReturn(Arrays.asList(UNDEFINED)); when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); - when(mockfunctionResolver.resolve(any())).thenReturn(functionExpressionBuilder); + when(mockfunctionResolver.resolve(any())).thenReturn( + Pair.of(functionSignature, functionExpressionBuilder)); when(mockMap.containsKey(any())).thenReturn(true); when(mockMap.get(any())).thenReturn(mockfunctionResolver); BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); @@ -89,7 +118,8 @@ void compile() { void resolve() { when(functionSignature.getFunctionName()).thenReturn(mockFunctionName); when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); - when(mockfunctionResolver.resolve(functionSignature)).thenReturn(functionExpressionBuilder); + when(mockfunctionResolver.resolve(functionSignature)).thenReturn( + Pair.of(functionSignature, functionExpressionBuilder)); when(mockMap.containsKey(mockFunctionName)).thenReturn(true); when(mockMap.get(mockFunctionName)).thenReturn(mockfunctionResolver); BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); @@ -98,6 +128,60 @@ void resolve() { assertEquals(functionExpressionBuilder, repo.resolve(functionSignature)); } + @Test + void resolve_should_not_cast_arguments_in_cast_function() { + when(mockExpression.toString()).thenReturn("string"); + FunctionImplementation function = + repo.resolve(registerFunctionResolver(CAST_TO_BOOLEAN.getName(), DATETIME, BOOLEAN)) + .apply(ImmutableList.of(mockExpression)); + assertEquals("cast_to_boolean(string)", function.toString()); + } + + @Test + void resolve_should_not_cast_arguments_if_same_type() { + when(mockFunctionName.getFunctionName()).thenReturn("mock"); + when(mockExpression.toString()).thenReturn("string"); + FunctionImplementation function = + repo.resolve(registerFunctionResolver(mockFunctionName, STRING, STRING)) + .apply(ImmutableList.of(mockExpression)); + assertEquals("mock(string)", function.toString()); + } + + @Test + void resolve_should_not_cast_arguments_if_both_numbers() { + when(mockFunctionName.getFunctionName()).thenReturn("mock"); + when(mockExpression.toString()).thenReturn("byte"); + FunctionImplementation function = + repo.resolve(registerFunctionResolver(mockFunctionName, BYTE, INTEGER)) + .apply(ImmutableList.of(mockExpression)); + assertEquals("mock(byte)", function.toString()); + } + + @Test + void resolve_should_cast_arguments() { + when(mockFunctionName.getFunctionName()).thenReturn("mock"); + when(mockExpression.toString()).thenReturn("string"); + when(mockExpression.type()).thenReturn(STRING); + + FunctionSignature signature = + registerFunctionResolver(mockFunctionName, STRING, BOOLEAN); + registerFunctionResolver(CAST_TO_BOOLEAN.getName(), STRING, STRING); + + FunctionImplementation function = + repo.resolve(signature) + .apply(ImmutableList.of(mockExpression)); + assertEquals("mock(cast_to_boolean(string))", function.toString()); + } + + @Test + void resolve_should_throw_exception_for_unsupported_conversion() { + ExpressionEvaluationException error = + assertThrows(ExpressionEvaluationException.class, () -> + repo.resolve(registerFunctionResolver(mockFunctionName, BYTE, STRUCT)) + .apply(ImmutableList.of(mockExpression))); + assertEquals(error.getMessage(), "Type conversion to type STRUCT is not supported"); + } + @Test @DisplayName("resolve unregistered function should throw exception") void resolve_unregistered() { @@ -109,4 +193,52 @@ void resolve_unregistered() { () -> repo.resolve(new FunctionSignature(FunctionName.of("unknown"), Arrays.asList()))); assertEquals("unsupported function name: unknown", exception.getMessage()); } + + private FunctionSignature registerFunctionResolver(FunctionName funcName, + ExprType sourceType, + ExprType targetType) { + FunctionSignature unresolvedSignature = new FunctionSignature( + funcName, ImmutableList.of(sourceType)); + FunctionSignature resolvedSignature = new FunctionSignature( + funcName, ImmutableList.of(targetType)); + + FunctionResolver funcResolver = mock(FunctionResolver.class); + FunctionBuilder funcBuilder = mock(FunctionBuilder.class); + + when(mockMap.containsKey(eq(funcName))).thenReturn(true); + when(mockMap.get(eq(funcName))).thenReturn(funcResolver); + when(funcResolver.resolve(eq(unresolvedSignature))).thenReturn( + Pair.of(resolvedSignature, funcBuilder)); + repo.register(funcResolver); + + // Relax unnecessary stubbing check because error case test doesn't call this + lenient().doAnswer(invocation -> + new FakeFunctionExpression(funcName, invocation.getArgument(0)) + ).when(funcBuilder).apply(any()); + return unresolvedSignature; + } + + private static class FakeFunctionExpression extends FunctionExpression { + + public FakeFunctionExpression(FunctionName functionName, List arguments) { + super(functionName, arguments); + } + + @Override + public ExprValue valueOf(Environment valueEnv) { + return null; + } + + @Override + public ExprType type() { + return null; + } + + @Override + public String toString() { + return getFunctionName().getFunctionName() + + "(" + StringUtils.join(getArguments(), ", ") + ")"; + } + } + } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/FunctionResolverTest.java b/core/src/test/java/org/opensearch/sql/expression/function/FunctionResolverTest.java index 1cd1e3756b1..6887837b356 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/FunctionResolverTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/FunctionResolverTest.java @@ -70,7 +70,7 @@ void resolve_function_signature_exactly_match() { FunctionResolver resolver = new FunctionResolver(functionName, ImmutableMap.of(exactlyMatchFS, exactlyMatchBuilder)); - assertEquals(exactlyMatchBuilder, resolver.resolve(functionSignature)); + assertEquals(exactlyMatchBuilder, resolver.resolve(functionSignature).getValue()); } @Test @@ -80,7 +80,7 @@ void resolve_function_signature_best_match() { FunctionResolver resolver = new FunctionResolver(functionName, ImmutableMap.of(bestMatchFS, bestMatchBuilder, leastMatchFS, leastMatchBuilder)); - assertEquals(bestMatchBuilder, resolver.resolve(functionSignature)); + assertEquals(bestMatchBuilder, resolver.resolve(functionSignature).getValue()); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java index dc57a13694d..9e678c8091b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java @@ -28,12 +28,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.data.type.WideningTypeRule.IMPOSSIBLE_WIDENING; import static org.opensearch.sql.data.type.WideningTypeRule.TYPE_EQUAL; @@ -70,6 +72,7 @@ class WideningTypeRuleTest { .put(LONG, FLOAT, 1) .put(LONG, DOUBLE, 2) .put(FLOAT, DOUBLE, 1) + .put(STRING, BOOLEAN, 1) .put(UNDEFINED, BYTE, 1) .put(UNDEFINED, SHORT, 2) .put(UNDEFINED, INTEGER, 3) diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index ffccf9a62e7..cc2acf57102 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -31,11 +31,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; @@ -103,6 +105,22 @@ void castToString(ExprValue value) { assertEquals(new ExprStringValue(value.value().toString()), expression.valueOf(null)); } + @ParameterizedTest(name = "castToByte({0})") + @MethodSource({"numberData"}) + void castToByte(ExprValue value) { + FunctionExpression expression = dsl.castByte(DSL.literal(value)); + assertEquals(BYTE, expression.type()); + assertEquals(new ExprByteValue(value.byteValue()), expression.valueOf(null)); + } + + @ParameterizedTest(name = "castToShort({0})") + @MethodSource({"numberData"}) + void castToShort(ExprValue value) { + FunctionExpression expression = dsl.castShort(DSL.literal(value)); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(value.shortValue()), expression.valueOf(null)); + } + @ParameterizedTest(name = "castToInt({0})") @MethodSource({"numberData"}) void castToInt(ExprValue value) { @@ -111,6 +129,20 @@ void castToInt(ExprValue value) { assertEquals(new ExprIntegerValue(value.integerValue()), expression.valueOf(null)); } + @Test + void castStringToByte() { + FunctionExpression expression = dsl.castByte(DSL.literal("100")); + assertEquals(BYTE, expression.type()); + assertEquals(new ExprByteValue(100), expression.valueOf(null)); + } + + @Test + void castStringToShort() { + FunctionExpression expression = dsl.castShort(DSL.literal("100")); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(100), expression.valueOf(null)); + } + @Test void castStringToInt() { FunctionExpression expression = dsl.castInt(DSL.literal("100")); @@ -124,6 +156,28 @@ void castStringToIntException() { assertThrows(RuntimeException.class, () -> expression.valueOf(null)); } + @Test + void castBooleanToByte() { + FunctionExpression expression = dsl.castByte(DSL.literal(true)); + assertEquals(BYTE, expression.type()); + assertEquals(new ExprByteValue(1), expression.valueOf(null)); + + expression = dsl.castByte(DSL.literal(false)); + assertEquals(BYTE, expression.type()); + assertEquals(new ExprByteValue(0), expression.valueOf(null)); + } + + @Test + void castBooleanToShort() { + FunctionExpression expression = dsl.castShort(DSL.literal(true)); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(1), expression.valueOf(null)); + + expression = dsl.castShort(DSL.literal(false)); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(0), expression.valueOf(null)); + } + @Test void castBooleanToInt() { FunctionExpression expression = dsl.castInt(DSL.literal(true)); diff --git a/docs/dev/TypeConversion.md b/docs/dev/TypeConversion.md new file mode 100644 index 00000000000..07697fed073 --- /dev/null +++ b/docs/dev/TypeConversion.md @@ -0,0 +1,172 @@ +# Data Type Conversion in SQL/PPL + +## 1.Overview + +### 1.1 Type Conversion + +Type conversion means conversion from one data type to another which has two aspects to consider: + +1. Whether the conversion is implicit or explicit (implicit conversion is often called coercion) +2. Whether the data is converted within the family or reinterpreted as another data type outside + +It’s common that strong typed language only supports little implicit conversions and no data reinterpretation. While languages with weak typing allows many implicit conversions and flexible reinterpretation. + +### 1.2 Problem Statement + +Currently, there are only 2 implicit conversions allowed which are defined by type hierarchy tree: + +1. Numeric type coercion: narrower numeric types are closer to the root on the top. For example, an integer is converted to a long integer automatically similar as in JAVA. +2. NULL literals: `UNDEFINED` type can be converted to any other so that NULL literal can be accepted by any expression at runtime. + +![Current type hierarchy](img/type-hierarchy-tree-old.png) + +However, more general conversions for non-numeric types are missing, such as conversions between string, bool and date types. The strict type check causes inconvenience and other problems discussed below. + + +--- +## 2.Requirements + +### 2.1 Use Cases + +The common use case and motivation include: + +1. *User-friendly*: Although it doesn’t matter for application or BI tool which can always follow the strict grammar rule, it’s more friendly and accessible to human by implicit type conversion, ex. `date > DATE('2020-06-01') => date > '2020-06-01'` +2. *Schema-on-read*: More importantly, implicit conversion from string is required for schema on read (stored as raw string on write and extract field(s) on read), ex. `regex ‘...’ | abs(a)` + +### 2.2 Functionalities + +Immediate: + +1. Implicit conversion between bool and string: https://github.com/opendistro-for-elasticsearch/sql/issues/1061 +2. Implicit conversion between date and string: https://github.com/opendistro-for-elasticsearch/sql/issues/1056 + +Future: + +1. Implicit conversion between string and more other types for regex command support + + +--- +## 3.Design + +### 3.1 Type Precedence + +Type precedence determines the direction of conversion when fields involved in an expression has different type from resolved signature. Before introducing it into our type system, let’s check how an expression is resolved to a function implementation and why type precedence is required. + +``` +Compiling time: + Expression: 1 = 1.0 + Unresolved signature: equal(INT, DOUBLE) + Resovled signature: equal(DOUBLE, DOUBLE) , distance=1 + Function builder: returns equal(DOUBLE, DOUBLE) impl +``` + +Now let’s follow the same idea to add support for conversion from `BOOLEAN` to `STRING`. Because all boolean values can be converted to a string (in other word string is “wider”), String type is made the parent of Boolean. However, this leads to wrong semantic as the following expression `false = ‘FALSE’` for example: + +``` +Compiling time: + Expression: false = 'FALSE' + Unresolved signature: equal(BOOL, STRING) + Resovled signature: equal(STRING, STRING) + Function builder: returns equal(STRING, STRING) impl + +Runtime: + Function impl: String.value(false).equals('FALSE') + Evaluation result: *false* +``` + +Therefore type precedence is supposed to be defined based on semantic expected rather than intuitive “width” of type. Now let’s reverse the direction and make Boolean the parent of String type. + +![New type hierarchy](img/type-hierarchy-tree-with-implicit-cast.png) + +``` +Compiling time: + Expression: false = 'FALSE' + Unresolved signature: equal(BOOL, STRING) + Resovled signature: equal(BOOL, BOOL) + Function builder: 1) returns equal(BOOL, cast_to_bool(STRING)) impl + 2) returns equal(BOOL, BOOL) impl +Runtime: + equal impl: false.equals(cast_to_bool('FALSE')) + cast_to_bool impl: Boolean.valueOf('FALSE') + Evaluation result: *true* +``` + +### 3.2 General Rules + +1. Implicit conversion is defined by type precedence which is represented by the type hierarchy tree. +2. Explicit conversion defines the complete set of conversion allowed. If no explicit conversion defined, implicit conversion should be impossible too. +3. On the other hand, if implicit conversion can occur between 2 types, then explicit conversion should be allowed too. +4. Conversion within a data type family is considered as conversion between different data representation and should be supported as much as possible. +5. Conversion across 2 data type families is considered as data reinterpretation and should be enabled with strong motivation. + +--- +## 4.Implementation + +### 4.1 Explicit Conversion + +Explicit conversion is defined as the set of `CAST` function implementation which includes all the conversions allowed between data types. Same as before, missing cast function is added and implemented by the conversion logic in `ExprType` class. + +```java +public class Cast extends UnresolvedExpression { + + private static final Map CONVERTED_TYPE_FUNCTION_NAME_MAP = + new ImmutableMap.Builder() + .put("string", CAST_TO_STRING.getName()) + .put("byte", CAST_TO_BYTE.getName()) + .put("short", CAST_TO_SHORT.getName()) + .put("int", CAST_TO_INT.getName()) + .put("integer", CAST_TO_INT.getName()) + .put("long", CAST_TO_LONG.getName()) + .put("float", CAST_TO_FLOAT.getName()) + .put("double", CAST_TO_DOUBLE.getName()) + .put("boolean", CAST_TO_BOOLEAN.getName()) + .put("date", CAST_TO_DATE.getName()) + .put("time", CAST_TO_TIME.getName()) + .put("timestamp", CAST_TO_TIMESTAMP.getName()) + .build(); +} +``` + +### 4.2 Implicit Conversion + +Implicit conversion and precedence are defined by the type hierarchy tree. The data type at the head of an arrow has higher precedence than the type at the tail. + +```java +public enum ExprCoreType implements ExprType { + UNKNOWN, + UNDEFINED, + + /** + * Numbers. + */ + BYTE(UNDEFINED), + SHORT(BYTE), + INTEGER(SHORT), + LONG(INTEGER), + FLOAT(LONG), + DOUBLE(FLOAT), + + STRING(UNDEFINED), + BOOLEAN(STRING), // PR: change STRING's parent to BOOLEAN + + /** + * Date. + */ + TIMESTAMP(UNDEFINED), + DATE(UNDEFINED), + TIME(UNDEFINED), + DATETIME(UNDEFINED), + INTERVAL(UNDEFINED), + + STRUCT(UNDEFINED), + ARRAY(UNDEFINED); +} +``` + +### 4.3 Type Casting Logic + +As with examples in section 3.1, the implementation is: + +1. Define all possible conversions in CAST function family. +2. Define implicit conversions by type hierarchy tree (auto implicit cast from child to parent) +3. During compile time, wrap original function builder by a new one which cast arguments to target type. diff --git a/docs/dev/img/type-hierarchy-tree-old.png b/docs/dev/img/type-hierarchy-tree-old.png new file mode 100644 index 0000000000000000000000000000000000000000..7add83f2867eaac109e5374936d84224035c64ec GIT binary patch literal 27195 zcmeFZXH--{)GY{z1SNwaQF4-;8&FA2rpXz}O^!`!5ETR^ibN4<5eXs+2&hPwpwI+S zqJWYcF`xtm6-DP<{JuBedh_0znYG@`njiB6Xu9vMTeogioxS(jr&G-_`t*l550R0P z(Hk1*T9T1bpvcI`-_TIOJK|T= z^739d_h`QmY0nT`sI+^Kdu&KBybgj7;HaPwFF$WTtk=II7_v66+Tj=q@a;rz9;4O^KPhd-?@OgoOJFD`><2hQSfo zK==!9!*4Sy_+<}2a_1?T+*NFRdUqhmCOe`FCUX zCfc!R_W&=QP#gI`lTa!u>*m{~abLFRdsI%^@d8#Drr1WBjmDex7hm z7!fp#u$P~^56&INPuA~0hAr66QZ5*)9UN+@5U%PO>ELRwsPM0mV0Y;3{zEW_k&}}9 z*NBQO&cibzBpPil>uwNXhBq;d4K%f|w{nj)_HytF3G%iz!-r#R4E-Dp^zhya%2t7J zXWB7=x>gp3F~L|*LuLEGK)+Ze6|X>dw63LMRD_wLzMZ0#f`TblUeO;1!w?Z|U|`^7 zgZ9>qurja=GS-d8C`MaY7-@SOng>Q3S*zlG9A$M)Lj24_l#G2tjbe?%gJN_|6%`#6 ze1fd-K2|<(gL=k}!5Fg`WzwV@G`gbHSxAqzy^id%KM;vOap=yaIshw zIe$g8UzD$5thcFFsJmmdFWOF9DcV#8W2a}0hra3s=vd+Paaao@Ya`PjHyLDeWUz`*#6g z+3JLcqdnl!tJ+%`dHJh2gqeie8~K@8`S_!P@SY~NRz^4_Q)Ly?5IEZt=WZykhsQ+e zT4VgMrg$^`&`6_j)eu8H+29a&y^v5_yKqZ06;zao6F*fmfJH}m>4cd0`b0!n=*xN=;G*@E18v-c9em(mC2g#Nou!J7j(MzVbTC{Q zuN>oT72$`M_4kVi!s_^AjKkqwJ4IEDy`{H}s;Om|rydUDZ);;@?;Gy!?`Ny;FN?D> zcCYw3HN+ghV=%1St$ zuvqt4LyWRFCNjWW2|3Z!LoV3S+cp{*a}RT6Ll0k+qKd9He59uzh0+O>$2j_0>Y*J2 zykdj&0x_|wfdO8y^JyE$`s*V@s$&nkr=Gi#t+IBEg)vUqOf@LVHWc1cK}CDXDSAf8 z$C|>9sf|_$f_(#C=_%+s1ls6@yT@4L6@9Hue64lNFs8O10g53w8y&-d5ZzD*J>7sX zF9&o~gq@Y4E(VQLmB(m%8=&m%gTiH@L*X6{TGm#&awceftRcos+rc9+QcDl^S)_T# zkPu}hB@=IV6)!KpC`Wm1*iYOebpm`t{E@Y)=-_Q>WP~@eHozF6!(w8Cd_26YyuyqX zwQ)+Z^2&iBhM|fM;V5~0fV>APBr4R{*2*WsP9LZ2jxqPw@xTN`VM9X0-Q9Hz^dds7 zeY|3oqILAbt$qD0Rqav64z{*Y;bw}ex;7}?5KA1g&&ci%bEvhdHC`!N)yEUrG8JI^ z*Za@Cc7NlA|NkSpDWJFG(A;EXd}M~YTGn`%&o^nCFgD|Tb}SWQ4^6(`NEZ;I_db-b zeJ8KQ(o4)rUiZ6qFdu!+z?2ZPA?*?2D`GhVlBINoIRIx6QD)&IX9-Q{^t^iVUAGu_w;@GURTU=cWrty> zErpK-X=)(%W;PiuA%RR6)8(1T1TkFT^ZsFHAzyt%qz0f`S&TW_lF%(p%x^6NNY}d|KUO!_sKoI`-T1PbXjDyO$PA- zedsA=26VQkph?t<1i3fP`T2`3gU&#tC%d-<^v#}*_zPxo1 z!4QtAtSbdf8stmZjQ^x&;LbWM(IbdBTVobPjI&3JzOT7PQ%X|g8&WZgk! z)bt^CY01{;^=k(YA2rAoF=gRR6A*PzTI&>q@#alftrKz+Aa6wv&pZxT*&39w6Fa^4 z%_^a;e#XY`jPU&VUp3|B<-|O3tEJw(+30!0$Nm%JKQH|GSxI_Pq%zp??nRO4leyS{ z_f4n&{Lo3|O(C$m##yvPe8|BbPvs7sbnvMfTYY&;MrAg_O5jXYxHyhQkI0}Q027KC z&tOa;!Xk?%Y5e{Db*u9Lqrv9V^Ml%S)3+w>+ub)YZ^o^Z`;L@mw%iivvuIm$Mmo%i zO23m6|An%R4doiQd5P}$DQBa6i3)jzu{)-@=YH)N4qlV8`FU42mF3X-)zg2(u4-(h zR693Y4iw19NI5<-zbLGcn`rqei(iLScJ&lX-W~tR>Z;)?$H({DwgygYeH$}7Y~T7q;Bn|M2Ch`@0>h#b-&^oPsT|_oL^}*>S1_R~s1o6AQU#9~_+t zp-zxQ506(_Re1Jg`d9zi`!)D}q)cadiQ?&>-A@xQpYH7xEt`o(nOr{yO9!vtX`HR# zGt4dQEASW|*Pzd>b0ToyeIa3C>>-+!d)z1-QMkZaKs{=W82Yxr^?cQ2mnMn(RGj@@ zN5qHGhKLWA&C8kU*3E~Weww&n@a@a2Q|c`PdIImcZx*z!jBMw*cXh?3&5AWF+hZdm zoy&W1`O)>m*?U@iBH0IWp9D?|^!N7zVJsH{`XwuM4 zb-NqNK@y47jMqeO#aY(xexJXvvY33_R)CV@y2C^7er&;+JDCr4DRqaxeHhC(=#qoU zmg`tH))o`)(1x!c=x+NutiJU&{AGV4hmJ^ykzdL=))6izGV^0|xgzLI<45<1(Q?C&3TfP~E>wyG zXbQP>0t1iQ-SoG{+2J3@#q0ftr5l#aVUM9@3AH3?a*x+Isd%~XUg}#c!OIFl`?KaLUda#D-@SU!O9m1*0!7_mNgW{*pt((Kl$i*zc_3}NMsuRi^~l^!_P z=CH>#Z&sf|@P2WF_NYV|J(scxHng^5*I&z)I3s*%i2fNpr!K4RaL|)kk^B>OwiS}q zd($1~T6fRkcajA}yW&?q%&?|7klcDt*jAg1j~WzmOICBKMci#+ivB!P;Xi@D(G076 zqJW~qAo%HC)WP7+c=C@`)UKyo&u!=LdKmuw$$0uVAFLfq&pQIo z&qU8w9Nk_tt__**xQ99SeKz|3AM@azF@}zm-R)16r*wjNU6}%5NWL)FIg!?8TkQ@V zmAosm(aRN*)WNv%P?)LCUhFZ_A!dZ9{`GvKq??)lA?OEPlhem=v5st!6n@9wA9 z`Jy&RN-GQyRf*Yf~V=h&hx(0li-0y(<0@n?bCHn;V&EN%OxfKExCm2y2JNu?rX27 zcd}B z6Jj!;z@%_{ohimI(LO&s&x}rEqpi7}z|<8@eJWj#4Elm%sb z`QX#TFE^4dWS-h^p;cU4>t^Si={tTw@(QPt<=^w$6Lu4&rvmIKDH%V0Q1Biyr=WJV z{Nd5EGZT<6H8t|;q0h@>vYY0BU({(-`Ei(HjVdFJMfKC}Ck94{UmOD*vJCa6)!Ljd zNv1xorJ|s4;D~$bFwOie@j3BYO80lR#<1mI)Nh7+4l)g_i%l#qzKk`?d>n9iZ~TF) ztCU1z$A!8^>I#v-?~O+fUFB3V!rtj&mvlPk@SDG*ZttzcFHeRkrOWcCW^P8$RrPj7 z%LCN17(5jr!t{{Xbyj%bn$-Pb?hP;Z_C{`YHkZ(Y%pRX@4H!k*i3v>glZ#2|gpHRy zPpXoijA)^lhThIx z=3;zzG=CF)(QdwPR+#fa+W)wS*VM#G2ajQp}NDUn}o6sg@a43j;%cRY# zu0ua`@40-NQ@$8qC*3DuhZe+jM;~(QQt%(gYxhWtpKwY0Ke+R(X^zv;cvp8<8gm|irkEwu|aQp6dsWVn7g}d$5fMZ$WDAOG8A#x8T zNgBO$G-&}pJ_9i7*hkSgyQ41S+g;Sh@6PjAq~B6x zmQ&&Cdga9WM1fx$7d77sQ1>o9cR0*0Z5_4zG_v#Jsl2-zmobVd*~z@I2JO{{=>OW$ z{dsxx(}m5)jZsU{#NWER+)s5rb7@nn-|Kd2jru(P_kFs`B}aOJPkZ0z?N0ys`to#+ zf)D2L(Y~2IH!10F6v}-<117ot{lL}nnDP|?mhSq>jTs$haqD}hK%M-FW}L@ zr2blG;RS={feDUKS@W3tNab1jj-@DPeFiuo^`4v@_yi(#uwkC zdi~wnW4FCNj8@ba-K(!lb?vm0^j#%nc{;jCXLdd^Dy+m#eDU;P!EA=U zL&YasO^-HkvB;*wl^y=P3p9+U`x0B?VonxMJ~W(g97O@Z3-6!e-%TE>t}QiK#LLl3+2S0 zKbEgzHx|0K0M!$p1DDYgg0OPUw#Kplj%iUjpy-x`^qJw=)@bAD25kQOvn%;_rKhe^ z-$0&{#;x#PxJL$N0F=)>aOVF=RYieI4p1axz6gMjy4kG8ZpZMF=*$GnY93sbz@L>u zM1?ISW$>PK1U6Nf(F#$2Wz6h!{_me(@8c!_0V8I}%KNtJw_fN}m17^Rey4w+-CQY%r zJ6~o`NI5m;o!{H7`g2s;S?qZ_djapsScASy9-xaMBx1I|NnXzll@MtFnGaw}>+kWF zdcG>rYVhi3u@MNEc~n(?=Z?{rx6KcePWp`s^t|(~Yu)sJ0R$FQ-a79{d!yd7Z(va7sbtS(daKpd7e#iof+@wrH`NE>1Fwb7 z1j|}yltVMFV(o3SRoamgcVriS1UzWB)w{x3ku>=7^k2y%64qJNym-wkBTp|}5O-)O zHzBB6OjbJxBJ}CwU%;XHKw2ac@Q*&X4n)111KRovdS$?jzSE$s@;0=G!@`O*#gn@B5(ssVJ*ZB6 zu>PK%z)5Pwn@}|<*e2G+chbzCxn#mHpvA;Te8l(Z0P{#wnCIi7LsFwR2nVPc?plrm z_#D4Tf|+pNvj@#E;9whbs83KQagC)x@BIaM(zs7GPrOSmS-vKq!po!A=WfsML5GdV z+}YU%Tt0At9Gb~(aeR5+cM>C>*Yu@zy+a}PQXzNIMdgKAa}wp-E&?xXC1)v@j-Vd2 z@7#!Z|4IvHjR+_Shd7*pX$s4ZrW0Vhwc)&27q5Bou9nUnDvlf91gPKipRl=CJ~_?T zYFY1rcBuEbJjz=xRK)P*1TNsDXfXUx(Jm#Ng9xIU5xIXQWJk#<` zG24?TDQE-G<3K3GyXJt`(%U~_iA~H;Vr27%WhVm~BTchS zU^QK)!<>VoI*~)xE&c}U(eFnkti?Q<52dF)-TinX=G%z=M7_P|fv~N9(OX2|>^8l^ zafh32ZtMc#K?IAB+t46J(jxF4ayj+f~;N_P{lcdgb@S4c|w_BKyV zT#7s?cIAkqeU?DqV5cHJFpWF$=F2K%-W$WbKX9V%)sH(x!H4pL`6LdpiIq??&K=JE zbES_7Vv_^C!!018=J#xC?lqv81elJC8M>dO*JWxr?JH)t{rQa@Hk>cE?lYGFHC9eC zLhs>#4XcPr79yrhAMB!Bym)ztm6ASCujtgdmbXAt`OnHy&G{uRH~81lhsgB)7*W1< zzf>5z6UK(}{&nl2C^<`Awx)mpwKrc`;qQgIQ7Er2T{~2A$**(q`EOv zUtQ#?U;-Xc(lg`CRiKMmnralOY1|aTaMc2c9_<>_Trj=6AfC}hovQp|48g1;;_wxS zl&`Yk^Guhr7h0e{vU;uKf+0H8B9bN|k0wY$qh6^g)JUY%jDYh?m!FjxbV@-3!P)Or z;n)NKoUC~E@QXJfyp9aHD;7Ysd0_zEds$(VQ<(0o{6d^gdg&|YdMrGe45$0 zWE#arNK{^gLys{EEyzHVf3~P=_@PhNp-;WTWgikC+_M@y!CAs_E=&;eFRJwkB@98I zWYqxc{ugB6#tIiI(u4pE8w@}Qx~%x85afRh9)BdlO0oUvbwP!3K@KAlwX-xYQhuDc zFA73{VIBKP4Cu2xX1z}RqF&27nup0Utfbz0wPlMu7e{AqF4j_$Z=@jK7 zvi%c$nR66J%hPk_CJV@xtzG8}y_yZfjIuo~y#h4s;0jk}z1luxSnc;<|I$bK4Fyf% z0zv=(1^)jb`hQ>+>4ozscxTY@YS`;t<&`>lc8Bd#X&h)pULe&W`|XD}PafzglvZg8 zb*j>r+9j%p<}T+6h~hW7Q#g1)t=(FEDKq`hr@*-x*WYpOmyl~m9PZnD3j`(ZqN2aN zO}KtS^tKFnZ&gQ!LO%{Ux3Apbbzm)o^E?D9Z7%oeTw5h(>+F8}AXuh%Wi(J(QROb> zu}ZD&z`A!>e^_LcCo&HxacP`mle9O_!oDm%IkZGMFcxceIqcq(zb5r_ z;`1!O| zwF>f?Now$w)znJHHHOY$iQ4BQQ`zJyH^VqXsj0nU@4yy-(WLlZ-^Y8HXd;g}5P*9( zeDeh46xhAOFV0w6StY7He${rYa<}z5bcNCM*C`;O>MmxjG36B%2fdWvnI7m+Sb5i_ zI-%0RQMtQ2EHv1I3?g4~AFDj~AXmzHhR46XbBOK)dCZ)dmdu)}8dYQ1OC^a^g18H;Yafa2&J?|S6*-J>97M(@O#hH(O zU(3l6_>r5JrwiGn69H2u5nhrU zY`Rx@7DyY2U=mpyGN(cC94!7U*s1#Xs_`J)kR!_gy_+@Nih|#FtY`oKuUr$0`{Pczu> z>&Y6?XF)3i`3L9wp3Fu$jhBJwtZ!TET%{Ycx4T_->g82xOVW>#6Ux)13gU;k_SUwI z?!&m*$k+X=w=VEbWH9h*h_RoE$X45d|F1zZMK{vr6~oet9`&n#fP>`_2$IQ+$1{DO zy=@L(nRE)txA({gN#eo`s-a-Ma%89sz_@()yCDWN$5 z&HBCBciiIU^6NyvEgw0!*0#1v%ZaJ;32dm%hsNKA5N-_Ftjd;z62H;%)M}S@M1l6EoN3Nl%aY z_I_k*=S|3<>xDgGNDn?~$vXXOO`#d`SQE#qQ}G|GDk)X!fpZh#NlD#Qi)4wDD>(<$ z;U0I`G`67sh@CUPOvjH>3DX7VCtM-C!K{D(ZZhe^Rz4J}qxm5cNUFrU-mAxt%h_hn zTxHXNKIT7qq=@?dHatIc;G?6MMQM^ejYCE9@$>fABpphJd6fL~+8}#q|AgZZ36Gw;6t*XX<8$Hj6;N_^ z%M~@Vee5dvO}EFYZ~5x!T1pGhrsN5iFO?v9WYS=PS?tc5_{&qj0-tq2-cCa2GMxo` zNV&E2aN_mYuP;<8Ac0%Cy*Y5=ffzq2YWYP^(9+-2b1<7LZ0_5BJV4|An@iY^#+8Qt z(G@Ngy$qWaf`MF;J%ej*?!;rVuuLv9NV`tF5VCY?iKv2f-fMMpfo|1pwhNcL6j)Kj zIW`#=XP&F;)^V;!CGBOcbI@^L=GvY1<_0X6y2d1?ms*bs+dad+u05lWk-jDMtksnZ zo#}egGx5y%Bd8|6jd5B0hFc6gxd{_U=DVVBB!kOJD2P}ENLIM^>7UsA6;#CgEXVta zhc{r|*s0tWK&Rb3MMZb4q*!7uW|il|RDIbloxm#aCQB7M#d7zN*LCha;{}RG-+hK; zTWI*hVDlfpiqC$raJboh>F}F{)Tx$*7lO(zQM^}F17}+vr!{}8fiw~#`G{NHF?oiw zX)*c^R_*#a+Z799!ijU=re~5?TGVXv6@m{r;fN7CEl5x&u=Qc z`g6d{a48wNQxlXzcKF&&e%aFeHD9|#Bb;8 zJ)TwDJXiF{9VF}ueGOh|+ej-59Y-e#HCKDTvIT*K8jTpJ=i zTYO7twz>67k#h_t@Z$VKqP<{1_zR`iccmdYe0TCaUwF#;;E?VxkZmT%1DDqWy&}80 zqJsE8ed*K7UetFY*4(;EBM#X$+FENxzJa!M9u{cIB!SUvCd&L(PpO zXLT)k(l9-{x&4i^IP#eqhnIDAb=i^o*NBG4bv9GDpQ+BBy|@FL%f0KD&NR{9wjebf z7j!v)OHbhqXgk6p8f~*-eLNEPuZ(IW$$R(?U6(V`T>gIH@7@#VxXtGh>Y2inVk=IC z+=N773e|VNAu;m#><-X%W=a3vUOq|k&Tg;M%4yuEW1ou@wY7;YGo!9IM)EI~cF|J{ ziv`rsB{2+;G081nJgDToR-HD^hvSWX&mWoF`GQ7;FOOLFj)MKMaIt)=)-);YddqEM zDs4dxb@J)@QqMKFZua)aW0dJx{CIN;qNMF-Ce$%5b>@^=owoqp=)(N9`MZN0uJpn1Z>9<0TFO$3QSGA5J8z zk2TRhuFJ3!kTkk1sN%!K`Y1dRgMU8E} zvr-EmxUjZw33Vd$p9bnM)by`2Z7|{Uo`2N^o;TT%1g~-#lK5=1pZ9)}?Jl5l3tDQx z4qEHmdnOJEH6%}R)|KgBy*GPyvIXsY_y>>fND?ipHr0ZH2A{wQ(rcKNb}y&g3g&}( zM=-R0wvu#usLq+j^z>kmc5%a~)w?VdfAo4*C!fbcSh|0y@5hkQ(x?sPt278uWN12ENYP* zUfd*QaIrZ4*i}&PO|N9^_0t&t9nRpvwSN0yV}X7H-MV~_9&>e9{oJMSg--Hq&Vw{! zZDX|Oel%Nm1(a6bQMm({^WMZjT7-!iGkOKL>d?RU4g>;c7< zuV1a-Bn7tbkgeEMs~0Ky<;XLnqwCdsKlQ3^&m7=B%?a3d#C-JIF;-m$=R_7ZBTHNV zBe!msUwXpj1M7a{Lv1gMk{`al6x_Z(^Vm4wKKn(zOZGxCN*1%X zaBwE;2qbW-Y5abl9n;P1tyOywF{g2Hso(8oVi;%OUn*=A8%k@_D=7vs>^c8mfaSv* zOqZU_E-^-x{M^128l{>wUm0Z{@su%*`+H%lvVe@yYx6g1Oz6xS)#*YVz}XB5*e8u3 z9@QVIe5wZfY{BW$$Cp*E`WkMk%7Dt#UjDMa42+AcQsVSi!3z1xFD|#b7^CpYKaKmq z&DzC0ahzg=<0BC(lCJT9Yv1O(MVSk)ZV;j23_2drqTVl5s0IP3WF_=f!iV(-RO77S zLkssMEndhtZBCJ67|gPlh0d(*zI)MwYFwK>1#IV+eS_#KoUPh7`L$RarxC5?OttZj znO(V`%a%BP^KT^?fGpn??0%FReKX|ybC+IiZH(U(uNxXq7o(@ z>M1IW^UXXgt2esEKA-u!&*q1v%XlD9yPNI->5~{KIJGV4WL*+8nGhIF@Dbu|Rns@eYY07CW zbrl2Ai~T~u$#gK`TUhL57331oVZ|qs36YZyOM>Cfg%8gp+6(m6{+YVkN@>E1`ZBuW z8-+BE&e>BnOhlsxxGVUd=D*8{kCBwsv83L{-sa#J+ftj}yA3PK{yn5pubI|eSD5$YW; z+OJAq7k)rdDUCpeQXpC_mVV4lu9Xq2}`A|;5V zCZNxH#!~vqgU8jB8?6BOJcCYsm@;uBx*IwB$;@O1N)w3#?-v+njuq}-qvyURlJ9hE z>cba6C;)sZn7I=lu)wLbE%eUNQ_YXDm5xzNvhqmK>U{G9x%-q?0#m z4<`7choAoab3#@5?9Q6wP`V!C$^h~H-(p{z)}t6nBXEVZSx7f6P!yuV@3ighycq?k zJ`G+0Fecm_xdCD#u_f|TIV2@J-i#(KrOTWull$+o+B+Oya*LzZ=cIsN+LW&6+qrC}+H@)Oz-aD| z_x@;xFFAfH>=nd$6o&p!bKB4GfsevCJ)dY0C<9>tKx&-gR9hPor^uEb0OUdhK7gHT zi}9ZVo1Fu~9r|ybekx&i4s6jMzXPYq2I6!7R;6jtodJ8|+M|Hi7;G%C6rppw-?Aaq zU<{Dm>;0?S;O%5}=!k1JDb{d!2mWHPuw`GS=Q4T{JZr7zwLaHA3Y*G@*AH%e3|Z{H zWZ8xfLwvUpYf~1}2Y%^ga#!{0qN-!QzQe7ze>VbFKEBSc21rW?5*HT24UpV#% z6ti&#c&2j#ryDNy4y+}ChnkqeEI9r^?sytOazrE|-caRglIEtD8SVbVFTTu4ef64V zd3wXy?_ajM56c*U@s|WbO)lalUhGM(i2O8JSdRYkFeO^O zTmE01@0QY5+Rt~VebUSwAPXa3GVJ)5ocSR@^3}DagoN$qH&0G1X659R$}9OYu`@Ab z>kfl+bKhj$x5TZO!s(@xHEZBny9@qbN>@tH-V9nP$IHhO#J~k%Z9sYi_Cc2mdyx~z zDNR5n$m?u*cO#%2eA`CvHGZu<JG#3j{MC&S95vsm`QlM~V43tcY`8rK4{!CUV~X3l{K!nkREN$mNaB zfV7o8*M2tn{CDt{ijyauF&8^q5kjQoLE&`IWn3t8P_HmwYJr^W@Ou+)+_#iSM?-Ku z9EZY&H4q=Ug|b{oZQxNHy{$uy82KUJenJrkQbm5JfebDuH`m6G2Kq}|N=0AD>E%0` z&K)TF0BQgneG=D<;OS!F$N!vZ6R}Xv^YWvhH@4f6?_cpZgNBz+>@SeH<@hx*QuA@Z z>tW41VCw*r_b5QTNT$Xv75{Ou9@O>K+B2==t3@gywile#yUf*k?0x)UZ=ibrH!{eQ zqci3Pcb7QD{+?!K@$7t%FmxCA7rTUDVx;T=Dmrsd_pp%L+Lr{b-LJd*i?$lKGT=k%!{yg3Vx4?Mq~1TG=Tyi2 zi_FWh_^?hqP5D!?`f%D6(KTH6*--P^g!mZH1qIW%^Qz2mbO(!i#Y4fxy(o2eemP2p zD1QD#3g*Pe>dSps?bIKbmrA&Hw0-;z4nONVWio9J9MicQCYUCo76)zmwdl1eez2q1 zm12om{P#|$aKtg3H6Q4{=uJpJbH@I4;;4Ps#e?9xqh-7LQtrdMMLBQ(v&nEf_Sb*negf(K~<=8)+U48l;(P#%fvHb z#asj##hF2+&U|YM6_~{smNlsp-1SKJt(*|Vw*?8%U*P^G&*i|i1ZWbhB=t12x53*eV_&Nft` zH5E_jLi%yPv@F#T?C78*XT@eFGGxfvFX3zJ)lZoY?i3H z51T|&n7KPqg%K81ug>+F^aEg!!_Y76pF|*|3T3JU0HgZ=he3L@APGJ4`PmTtUyX6` zS^yI1aE-%OfQ6^ezMDdBiu(ao$en$RwXvj-#)*2eZ)zXhE~@YaJuFr6FlKB&3+XA zlYZJ!NC0vc*T6z&3_d_%yISXlkoi-{cFBD*)*3@ly?q{nyNW}0`~w=<<@YC|UtKr( zcXAtDoUt2t%H*Tg>`~``v&vrxezW_l-%qxdEwpu@c!B4oZ{ds zvZ`~_Lcuw{nW*r0d4SVo(^>5wd^#xy0W|jPK;WnSDfQ5E_WMJ|m+xfLm(NCzXu{KA z*#K(J?b@8!TQ15}{KEZc^5(vyQi$m=PdC5Ur{}FHywr?O$FeZi-8z(Cx3r`^GffbHWSBJei>E=R4w-RD?W0% z?A&YDj%H0?SE2urDWHGhX;H-2SMyHqhWD2gtpUMFAX!`gbaJ;38*2BbRL*l)ycZYc z6nxwfJVzi7pW`CqFK zFTB-3@TnpYr?Tb)QeS~wMhGE>uLq6)?YuG~?X`K6B6@VDKCcRG+iVJ0zePS`E7b7qPoK>D~>7b zuRncW<3qMUAL|+|qfvIQtacPEP)&+Lgd-V*BHD++zhXt;Q{NCikbJA*QI3G9v2KS3 z@XdnY)Hh-X9@1G#Ek;@d&45wmEFH)i$z^Za!ANcTc6zg^_LW1&+AQzCn0iufIY#C` zvZ08S@a|2nNlVO+Nm@losD48T)wRWvKE&FM$P4giS4!c&sl)-w!Eo;vy$EBaH3+o& zKkwOX3%C(-evnX@^nvjFyOCa;P2F$rnLzv6E#k6>)a3>)IdtZ%?(B!F0)0Hg!B@K1 zWuF*?6LgW8KSUB@5kI9?Ake2joYKXXMmXl?7kbxFg^~WP7OMjPK9r{<$;19ImW_1t z%E~>rpD!Lw*Ez3b_nBH2L<#O#lGvo3?SVy{o5@RKmyw1-#XNo97+vsz`G1e70C3mf z*~fqsdMN@l@%W46?C79CN-g3&-&m#DlUXWtwSU%n#L1XIghEnU124QdZj00}2+Mgm zj|A375>zQZ$$zBuj{0~UAJoxYnfe}2et-l?#scM_8KP52JNDC$s7SqRPj63Tl9c}y`X|nni1??m#&DS zAX>@C_lQlmC;n3as1NkNsNLhkiJ0CHf()FQ?+jv-T~K%g!P08$-HQ_d;;%~ZLzc{V z@`0-i_@ZpPeIW~hTfD@6kA-bmr${;IMhP3$2m%7Pr;%DUlR|lqR6=EOtPxUhfS4S2 ze*6zbfGP(3;-$~1Ise>*-0rXM z>jiN?x2j%t{QYyb3Y@P7Bwde8Nz2)mNMR_v^R5CZl>^hfJ?Vsn)O2wZ%pKOXOnU#( z7d>(*Yg1wW=K1m>X_)Lh8wx4t(`+lz(j`EVC=Nz7xj|9P6G?cEg70*D`ox1a0dRh% z8Im+rZ-;oD1-DqGyq0}g;$yW%0qPZb8sTc5?u%Z(5hJ#ar_)kDPsrY1F>u7OMn`GD z+8fUCTF0>LuRTJB5c{v4GU7g{2B)&c4`5IeYB#$Qsg02&dS}9c+uN$dp^`Z8F^d8j zbTtuJeW5Z>hfO!%>JF~ezI3Sn_=VqLH;7&9NO?|Rl||WQ1i9^<1+e!qB(DpY_Pf%) zKha`o+^@64k=+^*FDRc)!;M~hPquWXHL4#9a7!ka-FSw9G-{JRy?*e3lBqFPr`&$! z9^?j%VL-irxmi}*%SFcR;wHdCfN;Q%(PkQ(NszuiT}n)7y64>7pjI38dFJ+30ah1k zwG~6=-fsJXjKhPX79wYnO#!(=lkizIJ8_g*UTCW2%H`+nzzeP3G}S1n-M4#STPo<> z>DKxk%#!RZe3LsH8()D(&=6GS@H&W-0das!x^_5oD*6KRSzNgX>BGA?jv3&S72xRg zXZ{9$X%VPhx5O$|{|#$}Tn)070h&}b&S_VNfP+Y9@;T0HZx{tcMRLIp0dZllS# zKza)-*HyK3yfo?!bv=hyq`a;nng{#Iou^Rh-IV!ugxu8e)dw+ys4rssD7$aO9QPpn zX(|DnzW0X?HHagHKMCU#BjY(yjK@Ld*t4VS7G-)>%QbXa zk1ugg3wT|r_SDZf#t(GdvTvykl9I_TR5xcvrT^0lfF}`|AHVbIPNJ>6awYO4*aE^| zI=Bm3{99zEEitN8**xl)e*>XGc*_B`xU4VfAYB!Ws56cMi6(T?EgP3vMjt%=bI-_* zxJ9VC2^HX2557B4OVB3yqN{Wjf!A~lf6k_S^YoccxB?#eDVj0rGk``n$b)Y%iT)!L z!+e`>{901EpKg8~;7#LxKn!<=na8?6^9KK0P#p1xH&4Qw;8~HZDF@SiwhSXg-U_EB zPYic)MP!3Ct1hg!z8Rbcn!u<%M!`;(b0gwx@ zDa_B#LZ%;SYTA7s9I)W1M!Z8WL#5p5AoZ&fo5#vMCV5jmAEeGr7rF&|U>OPppAHKT1 zeGBlyxpv*beXQH7{vU-r@F4yVDtaf7m?*V>GP(gD3qauBOGhb9j2`$eGI4@_0;NI- z7wI%qv`7+2wkSD3?#&&!!4jm1psr)0xQyE|k!m5ZIW#xr3nSiTj@}smf`z_Ewq#=u4EG=jr_;)Sw$@bZ{}5kmX$nU)}HrZ+f&F?!>20mOV1h6Qb@J{=`%+**>lwcI>2-R z)MYX-8*%}Ne8lHA`a+mW2DhItzEo z$KP*4U8kmGTHY^`|2$(OVSSemf?a-rORTuDamU_{Q-7}LRkgnI9r(5fB2X19RKHl5 z{(`S6$mUe?CxSqV{kgRwAG75}-U?p=;S=)*SXzaBy~ox}z=M9C=(qf!oM%Imh5)fv zz)SIBD~@-82Oit{R5K(5OKA=E~3W>U`O%+i#p38lZOng!b=t8Du)_Qm9@@N!Jwc&~PR_ zlW(d59;BzL3<1pGU!NcXrGhzv`k8lMg4h;>d`*Z=|3QZgsNpdJ;EDwo zZ;pC&aK8I>NFQ*YKJVO{!6A6~sHAye8n^{9h$Te`SFH_zJ&a}h1MykOTSZWKu`>4GKD{;zb8HH|jTo<{=^9_2{7%r=e8!5906#GI zEhT#@2fw{5uZD>3UlC8;ZsJdyYI_TCP1hJ%B40A{7M{-5D;>su2-ru8K>t7C>h@)U zl6j&;D=}*?dN#5Eftm8|-AQ*>Ns|d9y;V*-swf7;1v;$)G01|lv|rU@GXVLzlpC3_ zxjw$L3{`#BglZ)eb|5bss^K(|>q%J5gfRe#gD$ z@NNPz77{NA@7V%7+yFeI^4RT}`U$3i#6g%d6))aPaGI=qDf;A@x$`4Ptr##(MNIE`=y8Z1NQSd6! z&h=cyJ4*L)NEazN|NXN!v9u0fT_G{@9SZ1^b}vBj{0bQBtiQ&Au>w0)=bq*VFkJih zE9z&g?2zzxdI zwPNg=OC@SIU@P*F?FF6KUS!~T*qyur_9{zW_9Ve^-@Ua`c}iGDXC2Muce~pk3ODwD z0teK0s8`iE&lTw5@Z{JQkUY+JYh+(Rg zl^g)9%I2~Fmd9C=7|X9vtIk13<`WLV_*s@{fs1$do(x$#WwB}@0U|?t^jG#;4axEt zm0s5A;$FU5AynzEE>qhtN&m!fHbRDr61TAz$EgX~bTT3PDb5n{_fb!KQot#15v9$* z@DYsUIfz}md>!Dg;={C(v6v&C?# zT$#5a7$Hj3o))?^5$EBSI!w~U<}t?$WInXuV{YhEWBU-c5T1t$cLmdB?pALJ&hE0n z@8A=x9ORGY>*Ss9Gp0F?kL1P1HkHt}Cz2z1qE!hx?#!NfWt@1)-N61?qAt-ZmV6#W zqE@jzS7fiMelLIXWdvYWcBP^zHMDoWobcCrjn&=VRds}6d^qaf?$eE>DxjLbM2;NM za^3Rq0kpy3+cWS7{anGHW`dr~_-N$eHMiTfIW0m-6Lfx%{E`_cB>m`P2P^+8QJKuH zpCJo_+L`r42F}=dPD+zjM2t#iFtOHNv|rMkkXtGm*ULJtcS(kgug9F+>lk)qIY(5# z2f6E z%2Y{pY3sACOH4PvX2NDH9(+iylaqTzdz+9`l>fG9_F7=sxX9wv#}SgAC%bV9TqB&A z_9!pbo!lg+Jz?S!_B7kX;KTabueMeN%s!WX5xryfmuD);qmlKmHk1z#I7J~Ga(>D-<9okn|+L2qs|nx3a&rnA;-d`VWE0b zn@W*h{fJ=9SGJkx^PphIOCOAV_+_71hUG5HE z~`&~Q>kKC0;M{*%Xd;B{(yYj z%HdX>2h-VhL*6n=3D0&xxwJCDcsy;(ri{VcEDTWXwO}3X4vwFQKH)=A4rp=Zu=j-b z^22MbXGtOw7N1z=opb$Ih@O3_`<#8pLp~q4mx6UM*mHNvhHtIVbSO^^D%6K+I=%X3 z4&CF%*49Iv{=_{|q?+-NOhX6n0#AgPHgX;o0C>F(OE3OLBgyx)mW2Bnkg6DumxW9C ziwQ7r&43gh_4N+)&m}7FPE=ukqmzpR{6Q#E3qhpU>@KND!DY8E?sU^dCHjArQ#49o za{azx)ESf4;V3RA-vwWvH9d_^Zuux7;?x*h!hDDOdX4;Ipz{~h$VtLbBX{uJWbiFp z!$avd=N?5d1F=vgkNeJ6)Nrn$J*N^`cYfqVZ~7IE09#ZgV+8;p2Z2Z~2t-==OxXRc zQ4vqoglh1wyns$sEDCNxZbV4tn)7lF8XO!PdiX`)wl3e&a5P|d5~%>@^4 zNDYJrED4KS*a2UA@y8_!v<#;~oMhlMJMe8|)bLVpxEvfFOel0p*P4ODc?8k5{>xbj zJ%L_|sn9D3T|i`&JcD7I1DYz+?8gDZEJ0?Vyuh^X#AO(?hS9FYk^HX)kwYR9j2uspDh3G7+6&)3YIMz@ z;B$r+A?od&6AzT1kDuoQ7O)Hk6emVrL~~9Ig!c?mV63~b=l}}tOpsaHYNE6aQ@A=$ zcHdBVVpUAr_E?*M#v<<|8~FLN5WD@MX&@NakJ__OJFl(6Gc3Z9W%4^@Qf_)rP(@4N z8Ow7ZHcf~?|Nbt%cTyCac{Af*aT^hsUa<(GY0TE`+H~Pii0!W)-j}n@&5ZsNykCAf zDh42O!%A-217|eT66|$%hv#i@sA1qVu0I3CN#3|QaoDy6n5;YqR0@O+3#@{>9`jz< znSS+Z<;{t@QA3!G`u7Vn#DV40qJdTJ{C-sf(_Fex1E4$fgEq1n10Zg1-bZQ^ugc!EzAu!`ny&vGM{3mPo*f7awOi6pQ>C?MH%RBJm_ZshD zy*~rRl;C`kyE>6{MgTTb{oMH;EBanqV?zF=w1`*e+}{qapSu^mOc9uk3`_pFGI9;4 z_yhkey8Za-$r(pcM`d`o4y>B5<5v6shg2L$X@usMhbb54hk^NTM98Qde`bQ>P5!Me zTUE-RQe`Ir)Bnib9{`6>gZ7B|84(KQTCZ0M?;*AWO7(q}yx{b-Qwy4xMA>Vh&;cGU zDIUvY{c;bQe-iIsxN-v>nv=s6p20;V#1j^_=Qc_Y1 z&_q=PH$!5{Ado1xY-`F^0Jr~7Clira(wt#2YE!|O?7%~xlL=a?A6>f!__UGQcjv4M zJoE5H>__29IzY7n zw8j^K*Nt4Jm*@;`5#yD%8xYTF0QoArQKle3wntY9x6X9Nb0Xha9`H60Tp?WP_i?Kj zz>J2qyBc6FH<}hexK%r5MZ_Pqi6Ajbe8)7h^sf9$k+?tNbKHk2(n6v4Is$Upd-{y0 zuK7+p%Dcc-Ch0TYY~0JjBQG@RhXgUjIeTo(R$Kh1jiruUi%09u>KO6Z{ygaYaF)>S zTb6l&nfGwAr&}&Wp&Dg`HNnczZ8|EDh+&bSs-$H=(~m4%R@^tiE47yN2n~@{(BmGO z){xLttz$dA)E~`_$A=0sGXZPOCL<3rYR19jCNRuC3f1{EN#cK~-O^Dmm?oD7gXRJY4ag?!ElO)AJf!UP4A; z(|XLSKmurFsoCU6+gspU!kz>3j|K>gPQs=<5Lv zDWsS*jy@T;V-#KkE2(EVo4n*iVn!jdLS3DIY6_FhdvL~9)KPVlgx+&dbzOICk4jho z(Iw|`xl6^uTZN2~;et|cDF*lh0;R3YpZ~#g!umfPwl$`L6O*Z3&N2qyaI0|ZB^KZ9DuU~Nic+( zGHtJe&b5MtjW5XKS?)faRenz&(o<%7CeL&IBuQQ;AgOU5>t3+KZLfeqL5n_?!}mCi zdO%SUFU*V^x0pi$*yGeh3V>1Pm*3x3Fmm?S8#Ua4^`g=BU@%NX?g|JUMx(4hrn)g@62mu=NX{6xJUU~J%*)G=Iaxaj;0@4eS_iHjtq3_$0Fcrar)e0#}s@iCxCx_8uxD$^Gw2*^x61!e4oGO!Bg z%)f3hZ#g;%+S#Zf`EY%~!u7+@F1LGqm(yHwAOjFk@rQfLn=RHIA>tPM4U!Aczp1Wm zhGHB4#(^i#P~UKmSi38EUGJP!bM@mX6XjS7fnLr)T8jaXUGV~{!WvM4uyv$kq7I?` z?k!Rjs2u;umBzKEP&sKt-wtk#@|!;Ek;;EwCfC&BN6y5&v9wR@=ogK2gmD{qac6zM z{%J@P%6|b!W}bq})|Zp=3PwUD=%NB41q0LA+U_AA^&RL1jtN4MZT-U7eQxzf6ao5b5^9 zEkDx)_!nZLhKj-aXk=za@xj%NsKRR0)l|SGzUea-%o^&Jzx#o{+8Y>GO39k?c>M>ktJf=mnJp;@9ktTl^u_*tU-dmcVAd{GCe`p&l zxCluataCqlb{qg_X;{?c9~%@%pxy|5v;;6-oOljpSYB;9%mI2-JNHQjX;BruM&z9b zAzJvlP8?e;Y^KP-I6S*J+n0l6^^k(J*{V7UD}Aii8E_5oSDq-+Mtv-N7`RA!v=tCs zBC~({C?B!87|iTR{3>R!u?uP^5}$!ITNZ70-f!;T>SG7bJhi|?+9V<6rC%k7q z1K|7INOAbG5!|2{?3WNSci)*z>jgE4U_;Qy<0wNw?IMJ!m3pPMw@weC{t?xWr{N(8 za5Qh&`K`2_U=cH(H_2Bv1+BEGq|S-3@55g~4<4HWRDY4ov07aau%(sS$AtGgxIUA% zIq-ex=-?fPhb(bQr~9?WFa8cg_VDDOy%OddcPGw8GNmC`xIeHR(%BbRzQpx3L)?Iw zCV17`wVf1+-I3VQ8mBmvLbv?a=VAhF49uC8&M`^{Knl&>?80^GI!M}u4-#llbLk{( zlPc4E0FsdDG=rTWnBTOSp`y>VuSgb*?uxG(3f7oEcvL*5ZOUnP17^*ntckf%u!)<1 z*FG616+E${o?m-J#=cR$PG=6>j8>ijP}YI(Un=KBP~IF$uPPm+fYja@!uaL_08G3| zDq>{4UIQgt#n}TCHx7F;wg$y9t$lKh;CnWQT2)%60R&ue!U9nqQ4GwnKy2KOW~Yoh zRkii8#q_{E$F^cw+{pm@o<@Qm-Z zNrOLSeR{y(dRUW}z+G<{KL_#Q$bkTq3^$ddl*)wdtzY= z8UZqjR=YU806~4xn(RC9us>!4gIKvp+9LSj95MEHi>i{6)#~Zh1~w?$j((nGa&WYV zYW&{W7U0$b<^K70LpklW10Ph|nIh|DaL2Z0lGouAV-2LjGDumJL}W@^8Oxsj4+EGV zaYrxZRrSRD#@iHu!GQ`c1ACz8(8lB!rP{$l+j^Fj_3vRfj0QoC$hY9Ve}AbDY(7gA zCB>wF4w?`r9CW#Vs_E|u&U7&vlb3~9mcu!zW@KzOLP``;181~XZ-2*%&rj9|MFyODpth~`8F|7*bD zSXvNxDb!>numI~{Ye84wSYUKAEr7}l046o0{cA~vr>hYo7|n=ug|m=BG2REg5b(bZ zFakjT*XP9$)HFaRG-WIH;=)16#wN$^O=NE?C^%)}?gAtb=a zmSh*la*Q$c@<2LrJX{bVG*@RQ0v6>RW$I3kj}DKIiMKaI*wWasuJ#eO9>(!_6p;vQ ziVF?231P#%&_PaGFf<)*6hVTqSwS{vJ4Y;*8XIa$4~Ypk#voy?EV8!;(hct(M0X7e zL(n~a0tik>gg1i~PO@=fpb0QfI5B`i_wWgdaKf-mVmz@XE^ODpz);Yxy(7{YON_O4 zfQKO2m>_#YpJ0r4xQmAa!`mz#(vG?>KK9DkPSLCb69&(cX3uVV*ugHW>RbgpCbo z1INHb8hMjLOspA1lt-u|SQls-8^QFn4>80>c*3H>DU_%fZ)>(sJe=)rAB41X$9lLs zaN=B}U1*MMvK<{3Vas-hlVaSk?hLf4DS?LYjC2f-4U1>8IH3#^5|!ZUO$r2W3ZnVI z0`QSB_AF$C9nO#u5rB(`^zdYn0@$YBWKIwr?;c1qjWzNPb;HBms9~XoNDRUwB*Zk# zDLS0s6=mWE8i~M~SO=gZBYZ*vW8t8r~-=FpOz~pvT*J zpo4;N(NQ5_NTG3LPfV;~R0!B-I4qbpQ*hCQgb#3G!-8!&6oO53I0|Qq4zZ^YqkQZf zoD2h?JDeFIQ6{D)jF7m1fN(-Uc!&cFXX9>2a0`efvFz;W4lW)caVCt2py)6Yq8lwX z3`U4!$AucXkb;QeAru#yEuG^S;*N4cP%)0~ap2tumLZ;L7j0@ALp5~u;zZiTLsy|M zys%&s5d!1lo#AL4(bSlmbM>GsactnCHp;6lrO^~*Rras}o=iz99@xa^SD8?bIfC!pLbfCRA(j^jy zWMYT}Cy&4|LSFJLcZ&dDrhlLRt)+}5coQeT%3`YrrxsyFmj+6jXGMN_Z14Db!838evun1#YdbA-M^od3y z(J%}w!5$mMiDyua9732;4hS^UI38v~kFbkGc(YI_FFO_kH0Mge1TpEZ0W1PO&M1I_ zA;1FdDJZm?k70y&uxB_e%$mutiNoMnZop$9y-*I`9v-nQA_8gS6BFWY4UeaAg2E{7 zOz>L!I3xo6aS4K)BpkBGe|Xpbfvfc~F>s7Ep%bG5QM52xLL2O5BU^?&7R!6nFF zn|ee>2btIbSBo&B#FA{WILN_7M@N~GqrBlk@dTo=5pbF4NLRa%KzCXY)&YbFQzX`! zN((Wf`?#9Kgn7X{J?y>hQ7CdG#RC_@!9~H4n0RmJShy3>(AbR`97XlQJCcKNRJbvo z;>jSu!l_iW9m5ufbcv6l$Fc3azzf*!ZjrVz9P8jvYLsml!zqdr$cbR!9c_^~7f&*l zY8Ol61c22OZ5kShkBw%ya$GZDA@onoWyozY=ourM4PTFVih*m#s3IWB-mbS2V^ z2-JuOD%zOh7#>0m4T>hB9fCNJMI*fI4C9Q*#$X-dNOrbZIy^MUnQh0gw?%}&$<81K zqwSrEMhun-oE5`yHAZ`5yq#lANr8^eFpPpoJh8y_~^huB9`e87>qDxa9DJArwE24(JKILO%5PA7`szlIFUrR*ibk7D7OfQ z=&)FfBZqEe8bjhRQH01qlVHa<4iV{vfl+C$IOjN@(D+cg9n(10#EC>P#l|_r(h>Av z_dt}lgS&TdkTaT1Go@kT@i8U%fuDwjQ2#jTYI`jL|KR1k-ce5PZM`rjDtNs zED|1K$Z>VHj*nrpS!7R}I0}^lW7*@q32~l*2vT^6iJJ>4Jd)(;ZAWw`x_iJQi5@JE zFuEfuF4V;a9*m7;pd3sbS)M3sqyyWP9)e-fIW}k)S1+m$7QqS&V!Ii8M=|V!T?x)G zvQeOm6VrxZ7zJz{kEFUg(maAFR3|bl(gmK`Ed|-y z$HX96WQ=v3kBu!Vip*lN-9tFu;K_g}XDlZs1Z%^z2_za5B0^yBI0}puY2*p!*buCH zCKcS}LPSP}5-84&Y$u{|ptlz%8WzrRg1lHLh=Hb#USUSI^spdoSSXoHp@&D?+l2?1 zm_|n8Y@EEv5w1bTSPaV%$>O-hux!JTZf<0zrvr*i04@mOrk)Z&Ii9iQOp-ep)Pz-Z%i*^XaFx)(epsiq> zn=29uN^wM6R-_w?92#umW*BFW@bNM>1&#>B=g4?(YX&3U&N|q{IMmn(AMO=Qr#RcY z7)Fw44jjCzDK<1d&YKVoGmZ#=$IxSfqCF{~LAXl*oo*V&!C=EN(Xq_fXf`D(j!1wZ zS(spZvLTb`#6mp`j1LjTAqLGUuIsj=H-53A&+%fp}-;4u}U5tKk zZNmoD4R$uxZfw8zB_g+o&wu{eAyw6%D_WB3`~t64m8R{+klAS(Q;8?v3Qptpi&es^ zj^c7fV@NN~SH3uJo!4(K-CM8=^ZQ`geB8Hhe`2`P33sT(uxpRwZx3X!1}4KS-mRMd zYTNgVGd!;I>gfgB?9JOnayE#3x0;$hvEyrijQF%xigl_WXUyW4HlL>m??O-1X&o~S zKB=DL8*B*i@p}~%>^I!G&|@^jIaB168+3676)q)wBw2&08-1j_?55E)Je_;Xh&Q$- zUEZ@)hO*V<3lr(gP@)L*i)`$Uu!vvD`S7ys^7O@v7v~(4&-gebt9Sd1uwH9U?c+;V z`cv3ubbjV$VK#5-lMd{+lW)&u0c&;vk%H)?) z<$ERU%=>t``kKJG-lc7(LK=K+BF?;X3rXop@KxO9!rlN;$Sio!J=@XU;@I37b;sD* zU3?Vgp$CsT@a+t!CqK*7yB+U74W%&UH)>k(TS|c;OEzU-TzJjG`3HX3Cl@LmjA9u| z)fI05U0$o{F&d0$EGks$+8~mXoDGI`=;-^`&N8Uo_sE`ir2MtJJzpqg3M^_{X)-z_Un_(dm=jD@fIJ!hBM z^Z#i2Yw_L04!nJSxr*|r=;7xJ*_L?@du}7TZUo#v-azh^S{SXrV7}ZBw_EmWVO9;* z9@zi*)s3*TGP^wVyYGmrnCo8kxshl3>JTY=b@uGR%9wYv!Y8zs z=O3S~b}F9Z=%#bcHONAIlI1-j!saKm__m8Q8!$bt`e@CUCe)t)J(apu;9eI^?m%?z zp|4}qyfbICN!5;air?f89#WexOS)6~=hw%(4kx9wn2TYPw>ZrEVy2y0zrKtdbYHRJ zIdYLMdt=@f1nuCUR1e-c6#sL7#BHsS191oZTA#7>n4dFqlm%>()pb-qDV-6ZYx-yO zD44}wc@g*OuxkZ+Q;G!U>-^I%zw1Sn_qy$Lzf@URNE@u~%~Cewb#0Fr`t0%`ecQH=z!(}^Ex(sv@v28(a=)x? zu)oQzL#nLTS}$a(J>~Pqq(A$ei!OOyD=j6}5R^F|TIP8tbxI`)o~6s^oi8jf3hL@1 z5iP0?zv9f7eExR#&w)M<*8IR}ZKt9Tnuxbtnp$SuV~cM+y*g!Ag4FX7Ax~6)JWg8E zc#t7?7I#Xqw=Up5uW^tt`0C?}7ShjeANPE``{z#W>caSq%UKFPju{~uo<$ax9o1V%I>>T@B850KrDZ426x0S!HLl_E1Rw~ zp^>qylj10R6wB)`gxeMwhiPvV6wT>YNpLiPrEYi<+uvC2ii+pU{*3pyX3-E%GI{Hg7x1jd~o zl!d{XY{yb_?<8FA{k|u${1uWY+iaC>HCgmF5;93DZ71IztaxV!*a<3_vr*GghqQ(E zw}2gRE#tJyn)SrZh_Vkh=%y;n32~oT^!9y;x1txGVgGE+9lY$hr*Ln~rF3rJ?l#Ay zwU=?Pdb>_-|MKU%e&4Ac+T^Vk{|eZ6jv5A$%369Lx1|9+uv?GJP-Q$(EmCDNS{Jl0 zx?etXrlIcZJeaa4uezy8y3F(OT2X!{rG(vh`Umg$J{ue~exF{1aP2BFi#r20hx_}x zt6wK#DpV|+Kw>J;hA=PUf3t>lI60-c;=fxFm2rpY>VhqF@)HH0fhRXK% zG~#QUUvFOBXCoeqtI*$?JM-JT)TF_tMH02tcK7(!9kpQzO1Y^_83D0j!4lR0svG&` z*7Cq$zc>uz6?D|tCgjiR$E4W;65;6|67T5p-g5UfNnuBxe}{@|U7_(I)#+4`iz5nf z%9~B%M$&%*HrA)sTm&A&udMY+?8nKg#-S!sswt6Dk^6c6XWTEjJ=NbliCKPo`P$oz zuV)PQYoGOHopFccC)B5wYp%>ol%T*&){vS!>?Zs_C&U za>Gk^)_#T=9H`*&&h%JUhb%m+k}7*&XdHGX-Qd;^VY8+Wx4-$_2)}<^taVK%@qi|l zfV{TRXLd)5m5sRTgYOdFZLwFUF1}sr-Z4_Md#HP<SiYxA> z_ztlz6(MM@SB}5mu{X0kYLGH$Zb40WKb=qoKQ#e;Y6B&G$NMSuJ7^1|GXffEAi*!C^CEMAYK!4h&- zOGw^+q({Ik%-b1$QUX{N9Bk(%PL38gI5_X*Hb-LTL#4rsf1-I?8)CRy|#>tNmD%549Z z*IOnYmo!+;B_9aV{c+)C>_qrp@z^96p3U3O)8sr=iL{=Rd)yz~ty;<6xAsKFP|;HV zlgj2pR@R;69;Y@XhrA!ZA$>@GYGeOcZPwoSuA&$zbM|Hs+fNAGl0ap(^hiXEH`h#n=4y~qQ^gT0dp+qgJ%-(}y&k!uGRB?>NbAF$Xzh9@HKUcHMcl8s^39R9gbQ}jHl z>BQ@Kr0jg%e{KZ>FiuiP%&>s39BOGo?z%MUA1daJzpeYB`xALfp7Bdxi8X4h1%9O_I;+Vobc4gQ&m+}3jl+sS5{ZIKH1f#$`>JWbZ36*R;#tAmJ!MW zlXj|j{;VF6XM$NJ=6vZ~-q=#Vzwl9e98rBUz$?mfwtHM&!O?5 zizMSg$=NPPLc#6rw_Ct2dtAen6ggVjKKG~?cqW62oA8^0CZR)k**$*TiNQ>>(%GFe zzLcvSGN|F!QfZx(DFF2ax+hQN>M>uG_gMZ`1yEloILn$}Gg6c)WwTAko<;t3$lE6gEivZBBIVA;F$yuw0?NX5^ z0ZeyL9TS5xB4?hVAYRoM+;(qvs}I;s{)XVb)Dqi44Lh*#@))y~V6_|pJ&+cN$+F}3 z25TZ98a$Yj>;N9}vigDtAsq}_81%u~dH$ScjxiVjP6!%YGk~<+S~-iy1$@4Au-3QdA_Nl>2EFnEo; zGBmjVZyK>ZJeT$6iJEoUrp+>uy=Y*>STIxdeXqC%>p|d!EDGS$tK9gv7hyqNqJ=0{ zbt7#MXA&`~ucxOsHW2vh>-&cIFQFF!tmj5Qvr`#BHrJLV{-o)7)s;uT9nwQ;AenE# zQan0|MdS$J+hrh!{%taWC@3ng8gXmsI5(*Ts;75(nru3FFP1UU zbYMkICI(Z_b1Hi zYJnb!w=zf^`B>vrTpg@Wj)h?PqZNIxI;;3EZ?(ckYEQSuyn8s+ky-`PpN8lKSI*jx z`Pr*CC-WMD`yK*ps0T)Njba1g#g?V6?XzpYK8~zS9{zUVqI1QO==bC9<2NI&eQm!p zegDL}bGj#PggixmXnnVvHdP z1vu{OhnKrRh_<9S4FJ!=>&ew{t+La&$E5I1eFfnO+2h%6nJ#=SgnlE z6zez|Iwl1dkD*sjp$elThEhWYPL_hF$!|BRjwPLoVi zAAOg`114yxZ z{QMe;aob=hT_Nja)xjfIw%XX_Ydb}oZu6mpF=OA4S5^J^ZIMMn)5_u){A=63weOfs60z15i>K?=q#Gk% zS*{wbWzQ`ZpijR0(Dre_B>G%${lY{0#9A3t@Aa8NODX)D+-<>nU^U8+(O-Xg4-ridVi=2Z0RGV4li0d}NyKyd7| zqH$;snDdy!PPcjrEf2T8IJEp|e5BMOq0*~9a3|CLjQ+*2#5(bR0QJ_*8SicE%21kD zrKjF~d+BnPnb7j@AM<5j=C_+NhLTE3{B=tLt}H5I#rHJZfGw?wKG`imP#r6_ojN|a z;j*yi_8N$6ppfgtt0P|Bc_^BZ&{v?BPixtnk$x7WEn6*Ca#S*mO(b-n6n_z<`|#aM z`+G{B4FQa+3)1mHmh&)rPJ37GMoITu?5J%hkRn%eR$djF#!;_r z3=&Wi6GpNPNZk~`f9#q-`nNp1)Ym!E>&p9{UYx20f@!gB!_1Po6#lqXLsWW+gYLe7 zv(l5PsZ23}{ueDQ_%@U-sRD8AC81}-`&V}l#+e1E83~BTs;Q;&OQzm11NeQg#8cPB z{j!JCulK`OiwTm4ryo7+Dea^(Ut`4|;=LMg#k})S*c<&%`k9oCLZL|#@b9QaYrQR7 zilTPM#snPLY9CbPaJIU~PP?2zK5vVQ z09RN1CroX`9|n58$($>=tnGMC_1g?sboE>AG2I%ZZHGCflQV+E;)LE&H_Pq55*GF3 zV60eM(WhfK;1at)V#!>LBvW$~(6SH4?%hsYjX%(sc>@*F(+sVY*4yS!kC*tr*>nBo z?jW_Lo39dg>6Eh1*YZ82ySWvurrZku4lR-sl~Gxt;LM{-SGDrBw>QYjb44Sbflm0)1~+h;|0(V)$c*jGmPJ1g*eAK)&Yd_aeA) zK17=*UcdJ?arI~Ko2M6M0eGnYA*=7@3?T{)TxQ)7J{^Tb{>;HoztUvKp<069s}qTH zZ2R4SZ&E@=s#3x#(8tICX7#^5I+^fQz6W`udxU$D0~TWca-q9y_dYFnM$qYZr;7Av zhIHeedSw8Q>cyi=*C_re=-1CWsnU3#(sfZeaZdjc7)DU(<15$YiM^u_j=Xg9%?K*_ zb*j*CW=P)isWbGTP3PFweZM1ZD9qo8I&JCz7xnBsc8j%%&ZY1t<<>4GK6JlK((-|R zr17g0CjIRvN$K1)_IA_km6esW{;oT;_PC}O=JGXjU8XZbzOTl}oI_k!(CBhj&$L*l z`H6efrkCLB%t1rxsd?JD+xfS{74{K9?)hcb=uG#)0u!=;`fHP@-@_Mdw;sL@_X!OR zb*qkpTBNj|i3+WLMZdoHI zqU>{}CBM0XdI;Fp{VT=c9&keVa&zHw6;#3y@pj71m#BWC+*xe<#uuFmQ)@>Cd!4fa z<`>wbMbp3%@}~GLPnl*XE4NQetG6Sagr&D`J_lS*yZ9$2J6&ndLi~tngvw-n@Z%3) znY4E2S#WH@(`HR6-yCzLo44e_l1;NK>y|XjL<|Xs1qY_{K$?;-V*)3fhZ1&$tGq|| z9q*1&jAw>SvWrm%p)~(Y&T|=*NWkWo*0(rE2ec5n()^jEwlr~kpjqGLo=IzAKg&Ssus8dS2^+3vG7XY8p1ReNY@YMeQsm9V^-vzAq!Ot_< z`yLKo-Ct#MQp&Z38La2_kF)D>Wfao}D#`r#JnPV!T`BnWdnLYeVd$c|flr#uo>C2i z#L~Gm-2x$yhCDO)Ais@=9lK5?0^T&!H|oK*&yxuswGb8ePi*bG`)4)6q6t)h!0w(t zI*>7Lo!iXe-P5z~qERS`zn8k1-(JS`fud2+-QBL0Y34uQTwsFaPZhExU)t0I|D!k35usrY~kahI|^{|Td(X%=e9pd*jM*TF8Ft5YxIKpOn3HKP@;0P1aCd6-Inl; zp|^Hd=hDBxTYB%FP$BH=x!f%^0tz(>kA4FlTg%wk9f1Gh1yNA*&;VdEm8b2Y$7D*1 zS}PS5?F3279}zCSGwW4UJRrIH05ERQ{QPj&wH}btyj86Kes1mOyKC!}hOomzW0F`x z&WeS~#K(+0^={=8uDs^^MumQU3GaV6Y@RD zeLr!t4=QGXB=bo>#m$HOT4s#<_5Gi_`x_+85|c{zs@H(zO7VnlLhZr3e!w$UHO0Pp zVqKc>p=IWtNcZgHvV9YVeT(wx+~iuvq52C2ufIrN0CtI07t%_uz?`tJvOmrHz5Mv> ze^Hex0M)6tSk9}%-+C9vt_PV)AIdJ?%#cVG6wCYbdwyTh0&VD@T+!_&js&y!2n*S& zR46qBWtUlqCw1G+(A|;vz2XI+Q-2;amIT+Dr?i`BgW4_MUuS&5yq+tL7`B{D`g7R8 zc4}c3psW0o(mJ6u{`sFT)mh7U3E!NkqNg^UE_V}GD?x>zRa+?P=<#RoK-O7nWXgEa z>=?8yEEbkgaeZK~11M0;R^}62rwayeIM)Yam8HF_gjCp8vIaht^I7snXurfC!P~s# zg@Q9w_gB6i>YeM{_(&BjCXy`=s>yr+uRj;uGwE1nd7$>r&rt6=|CF`gP>yDK_-D`S z2N^NSg`{V1CvUfLBkzgC_m?HD>0yqGK1KQgKE4*7%2u>I)O_Y5Od5qc%y@lWZe*in z!pPPAI>w`)xJt^#z^4=@PAy0U9--!RA1+XejTI0}OG}rr;hcx$^1Il-2TIfja;TN=Kt9`!~ z;dxxk5%?K*hp-qA5K4hec_g>e1hf+*Xd} zhCMyVMC#vja2~^<&i9l4M6M-Xj`_1N^3hlVX1|H zl>g;S|8c{PRNmU^ub{q7T&6prte9x|qjJUS??{#&1JtP zi~&boxjpig^`tqhw)f;8|L#G37y?uPpKOtR|D*!b0PxAoFs~7i26F=?mx%^%HfGG! zc?ik<02%xjZMP=6_xlUro}V&npel?1^{1*#jal3FVejpL@<(O+4*t3)E%x1C?N!9x zRqf^$l&S9TjIFZdn1(j1+sr-KEKf4)gIv^TYfRXwmdy9SCq8%OlRZjl($?|Q?e2a8 z)>BbcPw!hFD?8D!q$d)Oz0$IacLk~#xT)LPJhRcn+PJcC^hL?7a=Ww_+Twog2E@AH zbmoH&us8EgsKn#finv3kvV!O8JzjlM`p zyhWEm`8^YB{#ci;M1PFsyiMI|`Q(|jqJTKVP(e<`j0o>}f!vZ?pQE5YUtP?Q8oFFk z!#$YzLo3!AWhSl41OTG(=rO^lV}b#xjuMK`YF-^$c5>vkpHYpAb>RsdnobL$MT4rU z&*#p&gv)@-IRxTBVOQv_o|D-XnOm}FHYItC@Y`SRk#z%UUN`0!peT6#@W-NMJtyO} zh0L;mXyn9JC1K6{K~Q<#))I1e?dRn)gry&zHyH;~lJ5Yg>t5$?pb#h_rXVG(+6bJb zOA~$U{MOSF=(gDR=~u_B&nS(d(@wiwz$t4mRsEPhUvyGoFkUmR&b#x6V~pis{h@YA zRNvV>o==ZH{&8f}mP_HtkC!gjeqK2OaDhxjUC9X--js-d+)l;L&GznNC%Nab8E-e8 z+WISVzoHAeos_(ckShDx`QSt=+(vxZWlB%!5od4KH__7gNWRw#+LL$6Z`tNb%W$54 zfq7++?i~6tDy=Qh2gD9nz0nBs!^2m+myy;V(zuyM1*dy@Klrp4!LOW-e6}_qHqh&H zd}n_Cnd*1DJ{hKz&~Q!t!@S?b7jLpIzkXo6oIx*oHG3&DjOpBEJ3M|;*6eOC5t^iZUU!oqOcMR<+az-D#vwWm&Nc52(sZyo}6r7A7+Axg<&jGD>heVeCIwq)0P&w!Zg>5>{d)y`+I&H9BZ zWyw!Dt@t}#?X^%@+(4~0xwhU=A>*{#+*4I95Pn2w_H^72i7hMcNjJ;jy^4Qla6mR$ z2&aF>79%UPRhXcCLR}~0Tvd|?6nfj;JY6m@d*yV>lMd`g$<#Sv z*-BPP&B@G0RpuIyRlIfLH`REN?5uvjXbU+_X{q43*iO#R-ZHz~eDk(jr6+F)FSVWd zlvMd$js2ufSmB6}>DeoR&w)Ma>!gh?f2#x}#q+_ac{6XWey7tDb)CzFvG0bR>ORTM zI_jI_Fjv&?pVAjqw0!r8bMlL^*X5a8z7qRvMN#wq*6@a1ONC12=9F(OCvOBw*rmCU zqjI#MlJKH3NjGD9Z$(e98UQhQch9BDphoMDV)F+2Hq^IE$)SPhz*L+Z`h;sNy87$e z$g-*xw}kjzWfdhR_Zj}!TzaA?jipAeyDck5)b|?8E`l|^RKJlSW6*TTj9;XCx_4t+ ztG{#;IqKP*XUT3ELUFsFwJ8J1o&_S1Q}OPyRBXV+Ih%KNV~-I(7U? z^!X8f2444ipsgrF)oaJ};@hR54hmr{^QgS7tkA6P{I24c0g2r-k3Ac07F044ayw8< zozl;pzWd&v_q?UsHNU-yuCe>6D*03U8*<7g_{y=zcfh>9Q4g3u)qUCH>N7_U2pwJ@ zQ?J}!{_N;0t$Y3abceLrhLU%i4y$E(*FKC42HK_-m;5guSMTWN-&`4m*B_SMlDgzJ zO`mvRfB4o3J~?B1;eiLI%FKuckmShHc#2BpMjiKK6BM5-@ag$nX=x`lESDDP1 zPF(Dk85+Ijy9jTC!Q$r|GOlYt=FviJ?OrT%;nbG-3LoSk@ zHt0L`1e1$pHhnzbnEZ@SHs^fBV)DymYtt^;6P2m&CqXUL?r2^`_b7AO>XNM1#qaMk zbZOJ?y9Ulg<@@#&pRk+!Jm9ey+Yvkcec!V*V>CO+pmZpekvUjy(Y_=N>Y(|;TA1v5 zED!m?+C+#(stjzNL@$GvBE6PH2X=* zQNdtza@4~*!xTU+i1^t*!=-aIJ2o9sj8Ne-OTMpTlN?nfp{*hVs(svK<*8Xv^7VW6 zr8Z*H{8mt3=}V1vWK6bJ!sjgTbR2!-$3k_TNIA{m4G}7K0W)@%(*E3^qTKshP*F~g zPT|_*1|2Vh`Scc_06fN{(5{Q&DyFPEQ2E?I?Xn(I>Yhw%D#%Dn973G60qY^UMH8(tS%6gc zn^%L*NANVe+vHQJ9+N-L+Ewt=T&|2sDGv^^*~jfDUssUL=G~)o*?|sx*Sm1~f8SVL z?A&~^oVkH2|H8ro(ArfW1cj`;q9Vl{t;wx@$Si zMqbsmLq9&-rS z@$uRNZ7aJ%!`}{FELuooUf0UMk2E$*SE!-<+B!V3!m-Z{O1z~4PkmHu3W^u|)lGKg zLP|CO*)>eBWlE>Vw>&J|Gc&a4OyL&2-&br2F(p~gXRgdYx=%mxc`l&o51oxZegY6G zBxzw)f*zm=+;-EC_M8m7>~=AaKHjL8PBLQRKdWxe(H3MET5XAt)WPCXd{#Z_Lv zF7a5#vkQv{2G%8Tx4;IN5%c`|76w(5QyR>O!u}iE%dJz*7m7>rlJ^VH(@E({H_snm zWrHQVt`O7$qlW6vM3GUM8W~jSaVfL7jASRds{Q?=*?+)=bY>Nwh?fY1tqD!t;;!lQ12h8;T^jacigh+pH^0C&hwYri{1=KAd5$8D(7 z17^t-wo{}HV(lV|9g?F$Yd_K(UQ`1Xsa~%^tn@PIZU>`bGP?NThGDY_{|!)gvN}>i z^A}`LCY)lH79UXP#(`#=@8{lTL*?Y4_Qg_2XrT_YwK_)#r(PET$;F%JZyxnl!>m?r zQ6Fcnn=0Y;_?tpdx_@sOmPzIM%!FRh356^nuYBpNe8eD@mhE}Ll+;CAIi&bNVGJ^$ zOXigDKyP&h?8t}f6nm4?d{?4E1^zx{W0-sasg=&vjX%u-MRrKc05(eB+1qXZsSpl& zqz_1@pxo%; zux*5^$$^5W%cU0ZK<@ZVSUz>VujiC5%8oHWUvu|sn7V`3a#^KV=?17Z{mu0J8nA*J zzJ-`U6DEKG){y_8WPBFvQKm@diF~NSFCvT;Pi==P__*fYc#ZI#8@)c#Q~B1{i=faY zr)m;!lo!sLe~2_5i8JwLKn5C>854Z`ygK8Vzo~b`-kG7F5{CTAPJq!1vcW&Y7w@)@ zpYY`e*6IHuWpESfiu^~M1m2COv^N?bNl)S5lzC3>lqO^(sZW2s+}QOD0jX{O(^9t2 z^~1kny>G|g7`p69B-fLtX(XfR$HKf^p!d6O}+RAa2oUIK=+mYx{IO`79Ypp8pW>* z)VoWYP2#ITb1xi`m(;1C6a}L_)8xFrF(QDj0jfzLQWMGaRVfbP&Nb{;A@J8qtuJg} zG3D5~Z7Kb`A42REDC`CUeem8zK$Pv-ap+S@!A^JkK6i-7SW5(XI}?yR@4kTyWDq6` zGrReMnL&vMRJ7{t_%C@k!ba?Yco7nxJ!_?w#V^=1rRn#VK2|JA4I`!S9M_k9{e)Z$*ooJG(_ zos+gceR?-3$-6Nm_e>9@YlUbSb!<8>E$TC`H*a6k+NhUziDA;(Aq5lqxi!UQ0P-Zm zOU3IJ=5lqM%~8&OokV#>SBJD#&KP#A%0W&6s7N9Dj=4~E_Tf3BAo9n_+s{CKK4_Q+ zJfB^b5=tFN9&?Kst#*B`&C}V+W*(*N*CN{x;K;xkpyl;AE~;>T{;aMG`k15*u{%q7 zwmoIzvug+UUG%we3DPhp{#?+}J9y_-QTEKck!yPZS?@gEKzXDHDP$q}^j=(<cmHh$K5l^AsRGsZ@g%~Q8nttg{eS@~)kQ0(&R#ki zr4>`|6r5bSVX4wtVtqQ%l1zlU4!;6+0FdzI_tXpwoZ!YG5^@sag(tat?*4Z5X`)~3 zeyxN^QzT<@UiNs^{shUwrX2zoaHrA@e2&Yf=KDX*|{2P$V6V-45`TRUY1q!eK|5Q9dpJTV^e%{P!r=W^AL-_?P3 z&~#ahzr;CfRd9;p|MJqd61kw2^0?H%BH?T1>IcBFIrAMo;H;YvOz#l@Yl)bY`R4-4 ztjhUy>#q-O*JHHdC`#wvt6*bCW&;)fhk7zmMXE>lzbR>t_CQq6=FCiW9l(D&U?gNE z@Dj!$JwvLPd>5X-8Rt=vaZHI=!!C&6n(aIL?q?A@7k>C>bjy#q;v3gMZ}9bj58QZA zWoP8Kzp^t}tXUd^f0YPJM(W)t;fbkht#2>Q40M<$t^J^`{(vOQ2dO&p_Eh;@ZMfW- zfaJAnVZd8X4)TQkB-iKmR=WDvcRN!u)#m|)J)9{z+_Cb0!n6C{>q%JfL$Xipz4Nbu zfUFm@)uOUH@RV3*FVOt&F^l^!la>9yxs*=*b6=ZhUU zCg-QC>l82zf9M+WZusipmwgf*x^_wrK2ZKWZ<2yP<6U7spAKp7c;bt5-;5Cx^%nQhfK{_W=ko>rQVNTL8RJ=^+ z?{-(=KUVY8R`q&@)Rt4%0RQ8)Ab9>Ykoc22Qor3JADS`h6np4Pzzolq#7*V`4;^A6 zgcf+cQA_ViO^Pu3tB~02rODd~VJ1T7g(mK(F`Z@aqp64*U*mv#0xcj98rm(aEi8v1 zm#T*N!^c5luNwCv%C=u6GbL_(s+p@uzdw;-lz9Hs>?IpJtt2kksyw}={_?+A$J%y# zH!;o1gQ(K;#(&ija6>T$;#qyQ0iM!#>S6^0XL#>!d|jgTD?3>uat7?fz`1G;zq-3KgzQjCNG9h16`C?JwOk`7pwx z1%AezT8MlIbfAgJh1@4?X@WpzCqC0I00lzFPeS4S-1QK7|MGthwhA@FCz2&Mtb-8^ zbRKRz+U42AoD`^VzcVY=Uyq#hbb9K1D2VTM#aB#UR>P9N44BCOjkS^G!kb5a)BZ6?fXHGCO-#%{LkV-;IRHJfBp+i7VDuwS@@lrbPNaZmHXyFgeKJ{ zSK96-j^Ua)*wcc@`-^7O74AJPE^v|yUdL}x^>&eHj^M$O=#;9UeY#zl*90yJM8AOjUSZf5&P3#4AY{4>+g99r?CbMLDfg6YbV1UM7_ z-^ZB~FIr%PM_)>u&P+fCGQl==ux>1zM~pfR7X$5qr9kF}br>x#nTL2*osl}P8Yw9tca5n`#)O&$G%1A^-Asd5o1fXZ@IrBr0me_fF9Mni>siAI-juY5RlK#BJB z{4v#H>E|XPPs;xxJSyTqu2iuG7kPet@!IgqtLs=;Sg2pGk5K%ijLrIycynPc;Lgp( zk`xHPO6MZuY{ZXcam0ZV36$^mQ)DOH1k;XpFLICl{a!v>4%IJwZpLl>py3l!vkO(# z)kSbVAD~3G;#ULb>ZAkbd0hygaD1})aedaSXV0DqKpCtIYC3AGff}EbAr~yPY4PZO z=&WAudY-=PGJxGSeXQ6=UF&EcPbWcj@%57P>E|W}Z!*u!FBYBp+k?^S|9I|{x{goc z^>7mz6d&h>_9uo$bx&0BKl@f)c;D}jU2~!D-1z-L!Avu;y&4&;-M#uYgIST zasGSY4YG9(v=h<4|K^n4{+-Oc6p+g&(o7QRazwVi=B9>oMLK5^@ zGD`D5-LCJ6?Tcu7bR5X)%^8D%x;G_Byk@7NvNc`i@9D)6-QS|i8Hn!QEznpBxwmZW z(*~+~L~`NGfX!IKZhwWyf1FIWZhT=aeO-0EuQogwkUsyNF%WV>emrU#tMlEK(;e=5 zmZ2f;%Hsk&+7Ld`6jbd7&h0^hTl4QzsgS7aX3Z`jUn~H^wukqQZJ7nE|HH;VP{{=# zTy=23BOl1$I%2lJc$1^vEtbBR3WX7vPP-$)MqLA+OKJ>p83P5CGn$?N!b1Gu<;oJH zSBbyEX@JzPKKu>96cr9n64X{tK@!{AtdAa485ARzybqe;NJu-MW`mWtA&k@>fP}H1 zyLTi(Wyj#xohR*Yf*ybV+_z_k?N8uqAf#VuzAl6)4z;ZPP7zW`JADn-$Mb&y zjvNX%I6&w65D}BUkZS@J>Q93Q>g&(-iU_n(QyX|66ntVA)z=qf_k(tkhRcQk5~V@H z;0Hi5RMrEI23C>aH+B0bMN4=VRPE6@A_?Bf8$r7KjRMc_gujTR}C1mVs0u~I-M z(X@rbqU$_#{CMTQf)z(^V7ziNuK}I%9+M`G(%(zNe|8*G?~ZsqnMDPR%>KN^mkxhn z;n`eX(5nC$l*1YQTu^k`c`V!VCw^o>RNl}E9N1J}FH3b>Qv_!(h%4{y0cwM?UAfTN zvaSZKE+@d$vk(@{8#$O{f#XfZQA=XZjJtwb7wUcE;L0CxFmNQvMX(L@;_%AoA2Dz? z*^DWU2PYIHp~ZwW%T(`{RJH;Y{e6Akmu6xt#D}LrHm!!c0|4jFeXA?^1$z$qWk2?+Eiwu^-pF|All1M)g=~9}pTV}FBs>B7eKw>jT?tTjR{vrKpnM9DLg^~2VUAp#2=RJh2KdsKUGJ|v{Q<`07Tvi;p8gGJ zo7;l;JI7R1lKgPF8&j4ctprdRYkB%{s`L-ExM`5dM={O9{i;3Kp@yk`UGc1x>w^7dbE#a6Jh{HiRNn?Dx4`Wsde34I3o8BIAOAQpx%o)>B zfIlZctF#;6HoP&wRyLMhaqC$Bvx_yQZ!dJOMd(7KPS{)Sv}LcwrrrSyt#$9iR zQ8CFS0pkAA71d(qTp_i~%Bsi8Uw?m-{#Uq?+2Nlnqs6)P@(NWN^P5sfUt0ur3R_I} zZb0qt$u&R^_n=}&>jIvEB;P`I{*)35_7&EC_}ABkqB3yS?Sag#>i^IJD8WsJzi>g? zw1(Qn(EJ~-b^(wR6i$PrSH0LYV)%whPv_M6j0jz|!+>`1X+@vWX^Z{-Dd173m1D`F z&%LOkRP1ArrH_b;ubJC))w34cAAqppdsN{_EeKRcH<&u%^VU&FJ~`*;d&Hm(Vf6c^ z;H_AKGY_>J9CZ)_I7$6=Fo=oD>yaYF7+h2U7yCbvvOEBN`JePdcH4tV3aT``xl!a_ zj4(Alm&FHR6bfkz9&kOBHY>lG9CUG2sOLEP2^ci zH4TDuI&-<5ixoX~nfZFQV}RLwpwqa_{(FZ_EeM7G%f2C?<1*^&LxIBk-=X}^f}Xc` z_~4O&5Y9mD*g7RAc{2PzJqJk5S zS-&7&n8JLKU1k+u*&`y5P$~pu0&sNfI`s91@&kM%^#2m9;|<(pkW2J~BS#=6GaK{N zb!=@a%mHs$z_1(F1>wLB^ZI0p^m@X~ztuEO6r375apBc%@ZTdiy_i^Pu6?rq&f-hu zb(3V78A-GE{Htn%ssgg(`2ap19Tq|8+x|6L*0g^Y-;#Q_Xa=ncqiSJs0HCFnU0PaL zR}x928<0w;`9xzRAIVCtV_S8#k7?XA$L>}d5KzuD3D7AMArSQ~t-b~c|M$;;^|gTG zfC4~3tRwtZC98$AOaWY3#t999!5DzB|Bv?0G#<+S-{WDjWShv^&=^GaLJZ2%*q5m8 z&}uD7VhHU9Wl0!Awjq?LBvHw}w3x9kCE9FLh;CcDT8+=gyvlNQSq7<)kRH;1; zS=MJ#kOc0>BT>l{y8L(x8-@Ba-U%0v<+f~-%397|$t6gDqW|L3T8xj?eaS(rq$R2) zIZ<-3&Qi-Ef)MEdR^}r(+bN+F4}Z(4Z~(0a8Lch5+1Cy8@qy5ex{{3~xf4)WYd}4( zBPfeAk-aB2ypCAKz1)q-v$yD0K(H&sYUT2@l6uQwXi%Z}-tEPO<^K@`H}kQ1sLbq9 zi8`!#rwtNlxzUP`C5GQx35$VQ`~S#^v=IDY!jD=gm# zTNvc@bQQQg9?WTj!#u@&1IoSvqN~`R&BLCU2WW&{)n`tXFU~Z3!eUR{Hv=Xl%g$WW zIb6mYupcP|d}^VB^edBd>>`*|(B=zO-hH%7AvYFV<^7rkrN-HdzeLWeuU*#51cS#X z`_=9`-oFMeJf@{WX43F?1H3MLarF;I80GNd(s-Q=v^3aHGj(5Fp=X?|ac?^<3C>2`C#F-&%H5Dgj;rzH)o6QJ6+1uX-E^0 z4VJ*#PlN1aGt8N8LhZQazia38;qR{ccy)it$K@;!Sgh?L=Klt*bS#%|wVxz)9i=L2 z8pe4U^(RD#<*pI#-QC@*ugk(V+xq2pqtwX?2Gg4UNg}IV3f2#pX~MfoN-@gS0Vz1h z>)I#q2qh#Xtvl9VQw@6Q*1bg)<3*J$#8vn@+PUM~m$&(b*CFp#=v-=Mt;TSJ?5DYJ z)wEd}gmKnw+C|dj1A|%u^s!9^4mUg?mz93|3EaewZ;sy=`5OHV)Eow=(Ko^PJp-{K zCd~J4LN-Q^v*%8>Oh6}a4^+9JYPZ=}=Bo-$maCqsHS9YGgYT>>X)%*mz$l}+E{6B%oEEMA$ESuAej$;;>3aLyjS z!CL`g8s5+c3B1nVx#8gAwsZ4-BIm)&b-Zv1S+>-wrc5$N;9F7#f2G4MATeBs-LMcE z(m=2K0&NHPd+h`Z`sKD!IsROjjpKICf`%iOSL;Dc@5j*Be}-fwX6N5tSWOr%1m7Zn zcZ6H;U93#h))Q~!+6`u7k8Db~2b%xq{@U6k7+aJ|dO68m$}??-M^HxB;3L_`4DoT+!J>ybw7;>;DK*<6fY|J^-)+bs#fSgC$LwUSNF z+T2_oJAQ(JnfIq){iUErso+sZKsji>_rI3LFy1-=5xbsQXc2F_39$<{aHWYx(LgF6 zozxE;^`kQ=e*RkyF{VPNLB=m=bRXP2vBONky?o8=Qv(_9ddv_Ca?L=Ka97uZ7iMyH zeiKBp8WJ{JSBpw}6-|Z^%(re8YTHD*QshM~e|Bt7hZ=bAt1B{u=1@OPqom~ip0u}bt*bQL|*7Oe9P^d_hqG6RjMxkjrYu65|zfMoi(h_1~b?GL9)?n zZaKn>%w3!xUWGl9^?H(1=G8f*OXYY#X<2h281O9P51h=&O!*YttewFPHkrgqu$L|t$UByr1;XZ@^pE1twRhfY555|h8)0-?;b0o7`(xziM{ zLuf2dlLK};qbJ+g2A>fs#*tfiNy?gG<6@4*@Rn$YIMv17i$&9nVU;0M2_8@7hhQxC zzQv>77C^$bQ9|kklvR36)41Fe+@BH-B(;=yE1q<<15^h^bt5BJmHMlc zs#7-I!6sWh=`8)kYh8i;`43v(eqV2k`n>f|N(B`2nYo5ZwV?jJxgN{EzgDKoDn(6d ze__V%>uEovdGj{!%?*;*yeu|iG(lne6h9;Nrhn;Y@cXN)4?nmR4oi#fqLDHgM71-? z_0WU3h_d;NH&$LL9&hbC;@iWQdtBAjF7W*N+i5^=Xv(B!@@d9VPn~!yyn}P3Rj2>u zDUzbTURq{x*K!lmAbIKGxHvmlBLb_-F^Vr*=MElHX{;b{uOF#4KRwnQAF!;x8xmsN z!5QFWTz4xYQbgeDsNC|jA&PTzK4^ykhC71$akOo10MbgKJ34=EDs~ljZ=eHbXv&Ud zV~ZTPIY?di&f^EC6ur$o3cHOlmq5x2&$ICqdh`ie$lrxqR_ z$+e>n93$clxL5p8tNYq}Y&f*iKvl(U{I(Nr06$xXuXK;Dc9`(83xptEM;C%OLj#lQ z&xeK;QY&(6@Pqmr90ZqcQwgk_fHi6nsE`*ECZ?DLm~72+yfJ3n2Hj;SW>EorLWTix zsTc~3J*`on1JSTETes+gp=#mcV)J#{4)GyZC{V&`zqO{a9vy2sx<_b3qgkd2)jg@w zO9g&+Di2)8k;#s&>*2b4M^Ck(kiIZfnOQEL`Em(bQ1ssHoF<_LK^KCa0DiB|=8sO6ul+f}@NW0watqjL;8Vj;Z$ArnOqSCUmq+14X+U=MD zeVBT)3-A#tj)jAy3H(Afo-{b}e+UQp??Hq9J&L9JKh#LQ3?zxG zRe$a3ZIRsSqllDpC=QySprm!g9&JsR2Nz2UB=2_uU*u^-Tl8mR*y-ndzK%S%vHNSj7<(;Avx3Lxt9U0U=`DN=_>3Vy#^*{E$|JP1}I`n z()ELjw2pO=5T^2E_1rapTH1iMXcLP3(?Ly5+)SGV`$;pB5g>U?gO`^Hr>0o;;mqAL zcR_hbmstQ+oRxmOs5909>|<_O=b-%i2(-Z-pIu1!PqbEp#_Ti+aF1%dqACd>gx{T~ z*rE_W;>UcO=#xc_=Y^b?Kzz|aDu_+SR$EvoQaYaaUJ^2m+b`D_-w#XP;>NaVi#j)U z*R@3GHV`Dz<1vfI}m z5=D8O?C?H>d`A1`2Eu#zH9R3rgXUYf`QFFic z=b3F>-(YZO{aP%i7W8ulC@87JPk>GrtljmmjMyn_mQmpWh_r2fhr&*2IECF|9FgE| zlc2*m$_Z$dv{Xj)UlODkS~7GI5E4p3&<_~f$gSoTeYsmy?SLZaO);vWFNQo-+|14n zP`FqR8l`u4etjxv=yjbj?~cJw~+zj4Ia-dArhLO4*0UvwcihyD^mQ3s-N+=7MmxA&J7O}wvk6kA;jLrK&nUs8@BzpP!7Rx3r_cFIii*a={NwRfF2fB5A;oKC zVc^n`%wZY|Zy|=x*pv{u2~#CXgn3Hmqs?Ae1*d0~K*;$a?JQn+w!WQAWBPxf%}$iI ze{RNf7H=PcAy|C!)sp4{yzngid;B!k3W{lmoUq403Thq5cFphkI@Ue6HkL(3s?vpXrL=UYzP3oI=GBp|1&ClR3sr9_NJl%e&eE3=~LF-*^8|HKMR4 z=ucMdwl?;1IvGwD{g4#<{5ckfm9|S81v6*GpPacoo(djO^qFoL1O~qT;{{oy$*hSZm(wdq8JL z%<+GHTK2JYLu9rhsROmet}vDEtYd=FWWT0m@oG=)qhIU>F3Y>XJ!DS^u~6i}D3M{d zDUZ^T`9zxS%d; z4+F;GYxFI@!G;p|MuGGB&)}(CecSb1yq+Gyerlmzcg!4SA%IfeG3Chkdu}}czEP+gbUJcn6;TfyM{bbM;0Is4GiVaSin>UO!@Nq+zpg`yvxmBs#KTHK`2TT#COdf zD?_DI+#~@y_Bs?Fk*=2Po#4lDzjbm2y(bXOh;IL+aph^T9Ay*=sm% zeYlCo4Zcc6&t1I|jkKTwNgZ?ib#8LxgjOVB!ai~7xW9Os9M%Dyl-{*pCl!s;gMmMO z0gbZ0Zib2gK!VGZG(6h|R1$H&eV&5`Wz3>;^4QVlRp&D|0ZLkk8*C(Y18NpctDqDw zp1z4i)+UOu7duZc%Cp$)L>3aAeXV2>Ah=o7nZxhg-Vo#u(?v1Y66M1@8nj_htIgPh z5YVlAY2{KhX#kLeSC9%TYkcuRTnwSnGkznPl*O)(m}ek#o_ZgizgecxI1W2bL5lj|<6(i3Eh@wha^Ak7x{fDFF>~SR9)< z3v-msyp0!1qpFVKa$|HYVS0^nY}PzRtZcN<*7ml@*4f0J-d@@VW3G(q_Hvq$5e8w- z;AP|S3KYn2WLI(gn2c}#>xvPcFam3IA{uYvfcQZGVRH6(4K5OHGX^feWFW#I6@+rG z5QA5a>_yRZco&#AEomqSJ_DMZv}QC4%2!U-C_{~$jGAb~A45w1Taj;gAHemiO)4Lnq=8atNgb3}Nc!M*gy(JOOsd{lG5>U1{^r0H+Cr<1TPQvui7 z1Rx!Ch!|V!V%I)PJ|7p+Aawe;A-y0pDzw#40 zo<{cwavaQau}#-MWR5o<+3Mi2*F|gxI%(NmvIZ=@`-dCU(@PMjL;96+#RP{5nFuPhZI{AY47V z2OHjk0L~*AN~rwtzz#6|RsD0^av3;qxQ?_SPS+l+xaoOcAWq<~9KW1wL6`4m%hJ8r zBU7qPlvpZ!nka956A&0y8z!IK1}pr)r1M~=E(F^XV7YGsM^3)s1|VlhN7gzWD6r4z z6(y@^`9j>`XWU06&Mm)hg^?M0@~g5zA}{`FtE^Qej(y9V9;!9|Ayf>1&5SJ#{f&Td zPA2S`b0fW!=Z35_u_l9*U*G8j0V_$}4ZfN4ulYaF!oLhg3P>da;Mu9Vv(Q=dG;Azi zX84XSixlW86BO8Uu*B4Qz;T_**&Jba?rG&oXj$|6q+kp9eZwr0jJ!w!d@m~KKIdFpnIo|o(sYKKKWHViWK{3+(N|fM=TWc4pAmWEx#>OCeiQ2D#4P6 z+>PM5TlpP+S72F#x>fM^VT=l37l~LV2>pIkRpCRwol|fSBla9PuCP*Yhxk7yS_U6_ zVw(8l*E0ps+89`h3*H&K|MSTI&z;QF(kkZ`_ for more details. +2. Explicit conversion defines the complete set of conversion allowed. If no explicit conversion defined, implicit conversion should be impossible too. +3. On the other hand, if implicit conversion can occur between 2 types, then explicit conversion should be allowed too. +4. Conversion within a data type family is considered as conversion between different data representation and should be supported as much as possible. +5. Conversion across two data type families is considered as data reinterpretation and should be enabled with strong motivation. + +Type Conversion Matrix +---------------------- + +The following matrix illustrates the conversions allowed by our query engine for all the built-in data types as well as types provided by OpenSearch storage engine. + ++--------------+------------------------------------------------+---------+------------------------------+-----------------------------------------------+--------------------------+---------------------+ +| Data Types | Numeric Type Family | BOOLEAN | String Type Family | Datetime Type Family | OpenSearch Type Family | Complex Type Family | +| +------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| | BYTE | SHORT | INTEGER | LONG | FLOAT | DOUBLE | BOOLEAN | TEXT_KEYWORD | TEXT | STRING | TIMESTAMP | DATE | TIME | DATETIME | INTERVAL | GEO_POINT | IP | BINARY | STRUCT | ARRAY | ++==============+======+=======+=========+======+=======+========+=========+==============+======+========+===========+======+======+==========+==========+===========+=====+========+===========+=========+ +| UNDEFINED | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | IE | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| BYTE | N/A | IE | IE | IE | IE | IE | X | X | X | E | X | X | X | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| SHORT | E | N/A | IE | IE | IE | IE | X | X | X | E | X | X | X | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| INTEGER | E | E | N/A | IE | IE | IE | X | X | X | E | X | X | X | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| LONG | E | E | E | N/A | IE | IE | X | X | X | E | X | X | X | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| FLOAT | E | E | E | E | N/A | IE | X | X | X | E | X | X | X | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| DOUBLE | E | E | E | E | E | N/A | X | X | X | E | X | X | X | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| BOOLEAN | E | E | E | E | E | E | N/A | X | X | E | X | X | X | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| TEXT_KEYWORD | | | | | | | | N/A | | IE | | | | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| TEXT | | | | | | | | | N/A | IE | | | | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| STRING | E | E | E | E | E | E | IE | X | X | N/A | E | E | E | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| TIMESTAMP | X | X | X | X | X | X | X | X | X | E | N/A | | | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| DATE | X | X | X | X | X | X | X | X | X | E | | N/A | | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| TIME | X | X | X | X | X | X | X | X | X | E | | | N/A | X | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| DATETIME | X | X | X | X | X | X | X | X | X | E | | | | N/A | X | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| INTERVAL | X | X | X | X | X | X | X | X | X | E | | | | X | N/A | X | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| GEO_POINT | X | X | X | X | X | X | X | X | X | | X | X | X | X | X | N/A | X | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| IP | X | X | X | X | X | X | X | X | X | | X | X | X | X | X | X | N/A | X | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| BINARY | X | X | X | X | X | X | X | X | X | | X | X | X | X | X | X | X | N/A | X | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| STRUCT | X | X | X | X | X | X | X | X | X | | X | X | X | X | X | X | X | X | N/A | X | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ +| ARRAY | X | X | X | X | X | X | X | X | X | | X | X | X | X | X | X | X | X | X | N/A | ++--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ + +Note that: + +1. ``I`` means if implicit conversion will occur automatically. ``E`` stands for explicit conversion by ``CAST`` function. ``X`` for impossible to convert. Empty means not clear and need more test. +2. There is no ``UNDEFINED`` column because it's only for ``NULL`` literal at runtime and should not be present in function signature definition. +3. OpenSearch and complex types are not supported by ``CAST`` function, so it's impossible to convert a type to it for now. + +Examples +-------- + +Here are a few examples for implicit type conversion:: + + os> SELECT + ... 1 = 1.0, + ... 'True' = true; + fetched rows / total rows = 1/1 + +-----------+-----------------+ + | 1 = 1.0 | 'True' = true | + |-----------+-----------------| + | True | True | + +-----------+-----------------+ + +Here are a few examples for explicit type conversion:: + + os> SELECT + ... CAST(true AS INT), + ... CAST(1.2 AS STRING), + ... CAST('2021-06-10 00:00:00' AS TIMESTAMP); + fetched rows / total rows = 1/1 + +---------------------+-----------------------+--------------------------------------------+ + | CAST(true AS INT) | CAST(1.2 AS STRING) | CAST('2021-06-10 00:00:00' AS TIMESTAMP) | + |---------------------+-----------------------+--------------------------------------------| + | 1 | 1.2 | 2021-06-10 00:00:00 | + +---------------------+-----------------------+--------------------------------------------+ Undefined Data Type =================== @@ -248,7 +346,17 @@ A string is a sequence of characters enclosed in either single or double quotes. +-----------+-----------+-------------+-------------+ +Boolean Data Types +================== +A boolean can be represented by constant value ``TRUE`` or ``FALSE``. Besides, certain string representation is also accepted by function with boolean input. For example, string 'true', 'TRUE', 'false', 'FALSE' are all valid representation and can be converted to boolean implicitly or explicitly:: - - + os> SELECT + ... true, FALSE, + ... CAST('TRUE' AS boolean), CAST('false' AS boolean); + fetched rows / total rows = 1/1 + +--------+---------+---------------------------+----------------------------+ + | true | FALSE | CAST('TRUE' AS boolean) | CAST('false' AS boolean) | + |--------+---------+---------------------------+----------------------------| + | True | False | True | False | + +--------+---------+---------------------------+----------------------------+ diff --git a/integ-test/src/test/resources/correctness/expressions/cast.txt b/integ-test/src/test/resources/correctness/expressions/cast.txt index 4018a73f093..3556e0b7952 100644 --- a/integ-test/src/test/resources/correctness/expressions/cast.txt +++ b/integ-test/src/test/resources/correctness/expressions/cast.txt @@ -17,3 +17,7 @@ cast('01:01:01' as time) as castTime cast('true' as boolean) as castBool cast(1 as boolean) as castBool cast(cast(1 as string) as int) castCombine +false = 'False' as implicitCast +false = 'true' as implicitCast +'TRUE' = true as implicitCast +'false' = true as implicitCast diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 60d7f8c6844..2d402186885 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -45,16 +45,27 @@ @RequiredArgsConstructor public enum OpenSearchDataType implements ExprType { /** - * OpenSearch Text. + * OpenSearch Text. Rather than cast text to other types (STRING), leave it alone to prevent + * cast_to_string(OPENSEARCH_TEXT). * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/text.html */ - OPENSEARCH_TEXT(Collections.singletonList(STRING), "string"), + OPENSEARCH_TEXT(Collections.singletonList(STRING), "string") { + @Override + public boolean shouldCast(ExprType other) { + return false; + } + }, /** * OpenSearch multi-fields which has text and keyword. * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-fields.html */ - OPENSEARCH_TEXT_KEYWORD(Arrays.asList(STRING, OPENSEARCH_TEXT), "string"), + OPENSEARCH_TEXT_KEYWORD(Arrays.asList(STRING, OPENSEARCH_TEXT), "string") { + @Override + public boolean shouldCast(ExprType other) { + return false; + } + }, OPENSEARCH_IP(Arrays.asList(UNKNOWN), "ip"), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 825200ce009..49af26eb460 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -58,4 +58,10 @@ public void legacyTypeName() { assertEquals("text", OPENSEARCH_TEXT.legacyTypeName()); assertEquals("text", OPENSEARCH_TEXT_KEYWORD.legacyTypeName()); } + + @Test + public void testShouldCast() { + assertFalse(OPENSEARCH_TEXT.shouldCast(STRING)); + assertFalse(OPENSEARCH_TEXT_KEYWORD.shouldCast(STRING)); + } } From b3dfc49d8aecd06488c8fa075c8a528dffc49c64 Mon Sep 17 00:00:00 2001 From: Chloe Date: Thu, 29 Jul 2021 16:43:52 -0700 Subject: [PATCH 09/20] Support distinct count aggregation (#167) * Support construct AggregationResponseParser during Aggregator build stage (#108) * Support construct AggregationResponseParser during Aggregator build stage * modify the doc Signed-off-by: penghuo * support distinct count aggregation Signed-off-by: chloe-zh * fixed tests Signed-off-by: chloe-zh * Merge remote-tracking branch 'upstream/develop' into issue/#100 Signed-off-by: chloe-zh # Conflicts: # opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java * update Signed-off-by: chloe-zh * updated user doc Signed-off-by: chloe-zh * Update: support only count for distinct aggregations Signed-off-by: chloe-zh * Update doc; removed distinct start Signed-off-by: chloe-zh * Removed unnecessary methods Signed-off-by: chloe-zh * update Signed-off-by: chloe-zh * Impl stddev and variance function in SQL and PPL (#115) * impl variance frontend and backend * Support construct AggregationResponseParser during Aggregator build stage * add var and varp for PPL Signed-off-by: penghuo * add UT Signed-off-by: penghuo * fix UT Signed-off-by: penghuo * fix doc format Signed-off-by: penghuo * fix doc format Signed-off-by: penghuo * fix the doc Signed-off-by: penghuo * add stddev_samp and stddev_pop Signed-off-by: penghuo * fix UT coverage * address comments Signed-off-by: penghuo * Fix the aggregation filter missing in named aggregators (#123) * Take the condition expression as property to the named aggregator when wrapping the delegated aggregator Signed-off-by: chloe-zh * update Signed-off-by: chloe-zh * Added test case where filtered agg is not pushed down Signed-off-by: chloe-zh * update Signed-off-by: chloe-zh * update Signed-off-by: chloe-zh * update Signed-off-by: chloe-zh * modified comparison test Signed-off-by: chloe-zh * removed a comparison test and added it to aggregationIT Signed-off-by: chloe-zh * added ppl IT test cases; added window function test cases Signed-off-by: chloe-zh * moved distinct window function test cases to WindowsIT Signed-off-by: chloe-zh * added ut Signed-off-by: chloe-zh * update Signed-off-by: chloe-zh * update Signed-off-by: chloe-zh * addressed comments Signed-off-by: chloe-zh * added test cases to meet the coverage requirement Signed-off-by: chloe-zh * added test cases for distinct count map and array types Signed-off-by: chloe-zh Co-authored-by: Peng Huo --- .../sql/analysis/ExpressionAnalyzer.java | 5 +- .../org/opensearch/sql/ast/dsl/AstDSL.java | 11 +++- .../sql/ast/expression/AggregateFunction.java | 13 ++-- .../sql/data/model/ExprValueUtils.java | 8 +++ .../org/opensearch/sql/expression/DSL.java | 4 ++ .../expression/aggregation/Aggregator.java | 4 ++ .../aggregation/CountAggregator.java | 28 +++++++-- .../aggregation/NamedAggregator.java | 5 +- .../sql/analysis/ExpressionAnalyzerTest.java | 18 ++++++ .../sql/data/model/ExprValueUtilsTest.java | 4 +- .../aggregation/AggregationTest.java | 23 +++++++ .../aggregation/CountAggregatorTest.java | 32 ++++++++++ docs/user/dql/aggregations.rst | 26 ++++++++ docs/user/ppl/cmd/stats.rst | 15 +++++ .../opensearch/sql/ppl/StatsCommandIT.java | 13 ++++ .../org/opensearch/sql/sql/AggregationIT.java | 10 ++- .../opensearch/sql/sql/WindowFunctionIT.java | 48 +++++++++++++++ .../correctness/queries/aggregation.txt | 4 +- .../dsl/AggregationBuilderHelper.java | 6 +- .../dsl/BucketAggregationBuilder.java | 8 +-- .../dsl/MetricAggregationBuilder.java | 46 +++++++++++++- .../dsl/MetricAggregationBuilderTest.java | 61 ++++++++++++++++++- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + .../sql/ppl/parser/AstExpressionBuilder.java | 6 ++ .../ppl/parser/AstExpressionBuilderTest.java | 14 +++++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 6 +- .../sql/sql/parser/AstExpressionBuilder.java | 11 +++- .../sql/parser/AstAggregationBuilderTest.java | 13 ++++ .../sql/parser/AstExpressionBuilderTest.java | 17 ++++++ 29 files changed, 428 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index d5c1538b77f..933e68085aa 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -161,8 +161,9 @@ public Expression visitAggregateFunction(AggregateFunction node, AnalysisContext Expression arg = node.getField().accept(this, context); Aggregator aggregator = (Aggregator) repository.compile( builtinFunctionName.get().getName(), Collections.singletonList(arg)); - if (node.getCondition() != null) { - aggregator.condition(analyze(node.getCondition(), context)); + aggregator.distinct(node.getDistinct()); + if (node.condition() != null) { + aggregator.condition(analyze(node.condition(), context)); } return aggregator; } else { diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 7400ae20e6f..3b78483736c 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -211,7 +211,16 @@ public static UnresolvedExpression aggregate( public static UnresolvedExpression filteredAggregate( String func, UnresolvedExpression field, UnresolvedExpression condition) { - return new AggregateFunction(func, field, condition); + return new AggregateFunction(func, field).condition(condition); + } + + public static UnresolvedExpression distinctAggregate(String func, UnresolvedExpression field) { + return new AggregateFunction(func, field, true); + } + + public static UnresolvedExpression filteredDistinctCount( + String func, UnresolvedExpression field, UnresolvedExpression condition) { + return new AggregateFunction(func, field, true).condition(condition); } public static Function function(String funcName, UnresolvedExpression... funcArgs) { diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java b/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java index 8753e35ed9f..e909c46ee7a 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java @@ -28,9 +28,12 @@ import java.util.Collections; import java.util.List; +import javax.annotation.Nullable; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.common.utils.StringUtils; @@ -45,7 +48,10 @@ public class AggregateFunction extends UnresolvedExpression { private final String funcName; private final UnresolvedExpression field; private final List argList; + @Setter + @Accessors(fluent = true) private UnresolvedExpression condition; + private Boolean distinct = false; /** * Constructor. @@ -62,14 +68,13 @@ public AggregateFunction(String funcName, UnresolvedExpression field) { * Constructor. * @param funcName function name. * @param field {@link UnresolvedExpression}. - * @param condition condition in aggregation filter. + * @param distinct whether distinct field is specified or not. */ - public AggregateFunction(String funcName, UnresolvedExpression field, - UnresolvedExpression condition) { + public AggregateFunction(String funcName, UnresolvedExpression field, Boolean distinct) { this.funcName = funcName; this.field = field; this.argList = Collections.emptyList(); - this.condition = condition; + this.distinct = distinct; } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java index e2c5fb6a39f..b2172e54f16 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java @@ -157,6 +157,14 @@ public static ExprValue fromObjectValue(Object o, ExprCoreType type) { } } + public static Byte getByteValue(ExprValue exprValue) { + return exprValue.byteValue(); + } + + public static Short getShortValue(ExprValue exprValue) { + return exprValue.shortValue(); + } + public static Integer getIntegerValue(ExprValue exprValue) { return exprValue.integerValue(); } diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 42a49db2eef..f3e69575965 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -500,6 +500,10 @@ public Aggregator count(Expression... expressions) { return aggregate(BuiltinFunctionName.COUNT, expressions); } + public Aggregator distinctCount(Expression... expressions) { + return count(expressions).distinct(true); + } + public Aggregator varSamp(Expression... expressions) { return aggregate(BuiltinFunctionName.VARSAMP, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/Aggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/Aggregator.java index 80944172ea1..5328e11aadd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/Aggregator.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/Aggregator.java @@ -64,6 +64,10 @@ public abstract class Aggregator @Getter @Accessors(fluent = true) protected Expression condition; + @Setter + @Getter + @Accessors(fluent = true) + protected Boolean distinct = false; /** * Create an {@link AggregationState} which will be used for aggregation. diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/CountAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/CountAggregator.java index 3195bf39413..f1bd088967b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/CountAggregator.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/CountAggregator.java @@ -28,8 +28,10 @@ import static org.opensearch.sql.utils.ExpressionUtils.format; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; @@ -45,33 +47,51 @@ public CountAggregator(List arguments, ExprCoreType returnType) { @Override public CountAggregator.CountState create() { - return new CountState(); + return distinct ? new DistinctCountState() : new CountState(); } @Override protected CountState iterate(ExprValue value, CountState state) { - state.count++; + state.count(value); return state; } @Override public String toString() { - return String.format(Locale.ROOT, "count(%s)", format(getArguments())); + return distinct + ? String.format(Locale.ROOT, "count(distinct %s)", format(getArguments())) + : String.format(Locale.ROOT, "count(%s)", format(getArguments())); } /** * Count State. */ protected static class CountState implements AggregationState { - private int count; + protected int count; CountState() { this.count = 0; } + public void count(ExprValue value) { + count++; + } + @Override public ExprValue result() { return ExprValueUtils.integerValue(count); } } + + protected static class DistinctCountState extends CountState { + private final Set distinctValues = new HashSet<>(); + + @Override + public void count(ExprValue value) { + if (!distinctValues.contains(value)) { + distinctValues.add(value); + count++; + } + } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/NamedAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/NamedAggregator.java index 346bd2d28cd..92176e86483 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/NamedAggregator.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/NamedAggregator.java @@ -54,8 +54,8 @@ public class NamedAggregator extends Aggregator { /** * NamedAggregator. - * The aggregator properties {@link #condition} is inherited by named aggregator - * to avoid errors introduced by the property inconsistency. + * The aggregator properties {@link #condition} and {@link #distinct} + * are inherited by named aggregator to avoid errors introduced by the property inconsistency. * * @param name name * @param delegated delegated @@ -67,6 +67,7 @@ public NamedAggregator( this.name = name; this.delegated = delegated; this.condition = delegated.condition; + this.distinct = delegated.distinct; } @Override diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index 8cb7288273c..99a316fd851 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -300,6 +300,24 @@ public void variance_mapto_varPop() { ); } + @Test + public void distinct_count() { + assertAnalyzeEqual( + dsl.distinctCount(DSL.ref("integer_value", INTEGER)), + AstDSL.distinctAggregate("count", qualifiedName("integer_value")) + ); + } + + @Test + public void filtered_distinct_count() { + assertAnalyzeEqual( + dsl.distinctCount(DSL.ref("integer_value", INTEGER)) + .condition(dsl.greater(DSL.ref("integer_value", INTEGER), DSL.literal(1))), + AstDSL.filteredDistinctCount("count", qualifiedName("integer_value"), function( + ">", qualifiedName("integer_value"), intLiteral(1))) + ); + } + protected Expression analyze(UnresolvedExpression unresolvedExpression) { return expressionAnalyzer.analyze(unresolvedExpression, analysisContext); } diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java index a27d90f35d0..af2dbf22fc1 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprValueUtilsTest.java @@ -96,8 +96,8 @@ public class ExprValueUtilsTest { Lists.newArrayList(Iterables.concat(numberValues, nonNumberValues)); private static List> numberValueExtractor = Arrays.asList( - ExprValue::byteValue, - ExprValue::shortValue, + ExprValueUtils::getByteValue, + ExprValueUtils::getShortValue, ExprValueUtils::getIntegerValue, ExprValueUtils::getLongValue, ExprValueUtils::getFloatValue, diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java index cc2825858a2..1db33ac9d5f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/AggregationTest.java @@ -116,6 +116,29 @@ public class AggregationTest extends ExpressionTestBase { "timestamp_value", "2040-01-01 07:00:00"))); + protected static List tuples_with_duplicates = + Arrays.asList( + ExprValueUtils.tupleValue(ImmutableMap.of( + "integer_value", 1, + "double_value", 4d, + "struct_value", ImmutableMap.of("str", 1), + "array_value", ImmutableList.of(1))), + ExprValueUtils.tupleValue(ImmutableMap.of( + "integer_value", 1, + "double_value", 3d, + "struct_value", ImmutableMap.of("str", 1), + "array_value", ImmutableList.of(1))), + ExprValueUtils.tupleValue(ImmutableMap.of( + "integer_value", 2, + "double_value", 2d, + "struct_value", ImmutableMap.of("str", 2), + "array_value", ImmutableList.of(2))), + ExprValueUtils.tupleValue(ImmutableMap.of( + "integer_value", 3, + "double_value", 1d, + "struct_value", ImmutableMap.of("str1", 1), + "array_value", ImmutableList.of(1, 2)))); + protected static List tuples_with_null_and_missing = Arrays.asList( ExprValueUtils.tupleValue( diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/CountAggregatorTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/CountAggregatorTest.java index 0fdadfc692c..ee183dafcef 100644 --- a/core/src/test/java/org/opensearch/sql/expression/aggregation/CountAggregatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/CountAggregatorTest.java @@ -129,6 +129,35 @@ public void filtered_count() { assertEquals(3, result.value()); } + @Test + public void distinct_count() { + ExprValue result = aggregation(dsl.distinctCount(DSL.ref("integer_value", INTEGER)), + tuples_with_duplicates); + assertEquals(3, result.value()); + } + + @Test + public void filtered_distinct_count() { + ExprValue result = aggregation(dsl.distinctCount(DSL.ref("integer_value", INTEGER)) + .condition(dsl.greater(DSL.ref("double_value", DOUBLE), DSL.literal(1d))), + tuples_with_duplicates); + assertEquals(2, result.value()); + } + + @Test + public void distinct_count_map() { + ExprValue result = aggregation(dsl.distinctCount(DSL.ref("struct_value", STRUCT)), + tuples_with_duplicates); + assertEquals(3, result.value()); + } + + @Test + public void distinct_count_array() { + ExprValue result = aggregation(dsl.distinctCount(DSL.ref("array_value", ARRAY)), + tuples_with_duplicates); + assertEquals(3, result.value()); + } + @Test public void count_with_missing() { ExprValue result = aggregation(dsl.count(DSL.ref("integer_value", INTEGER)), @@ -166,6 +195,9 @@ public void valueOf() { public void test_to_string() { Aggregator countAggregator = dsl.count(DSL.ref("integer_value", INTEGER)); assertEquals("count(integer_value)", countAggregator.toString()); + + countAggregator = dsl.distinctCount(DSL.ref("integer_value", INTEGER)); + assertEquals("count(distinct integer_value)", countAggregator.toString()); } @Test diff --git a/docs/user/dql/aggregations.rst b/docs/user/dql/aggregations.rst index 1d6d1729815..275666e7bad 100644 --- a/docs/user/dql/aggregations.rst +++ b/docs/user/dql/aggregations.rst @@ -357,6 +357,19 @@ Example:: | 2.8613807855648994 | +--------------------+ +DISTINCT COUNT Aggregation +-------------------------- + +To get the count of distinct values of a field, you can add a keyword ``DISTINCT`` before the field in the count aggregation. Example:: + + os> SELECT COUNT(DISTINCT gender), COUNT(gender) FROM accounts; + fetched rows / total rows = 1/1 + +--------------------------+-----------------+ + | COUNT(DISTINCT gender) | COUNT(gender) | + |--------------------------+-----------------| + | 2 | 4 | + +--------------------------+-----------------+ + HAVING Clause ============= @@ -456,3 +469,16 @@ The ``FILTER`` clause can be used in aggregation functions without GROUP BY as w | 4 | 1 | +--------------+------------+ +Distinct count aggregate with FILTER +------------------------------------ + +The ``FILTER`` clause is also used in distinct count to do the filtering before count the distinct values of specific field. For example:: + + os> SELECT COUNT(DISTINCT firstname) FILTER(WHERE age > 30) AS distinct_count FROM accounts + fetched rows / total rows = 1/1 + +------------------+ + | distinct_count | + |------------------| + | 3 | + +------------------+ + diff --git a/docs/user/ppl/cmd/stats.rst b/docs/user/ppl/cmd/stats.rst index f6dad255efc..ee91d86d147 100644 --- a/docs/user/ppl/cmd/stats.rst +++ b/docs/user/ppl/cmd/stats.rst @@ -302,3 +302,18 @@ PPL query:: | 36 | 32 | M | +------------+------------+----------+ +Example 7: Calculate the distinct count of a field +================================================== + +To get the count of distinct values of a field, you can use ``DISTINCT_COUNT`` (or ``DC``) function instead of ``COUNT``. The example calculates both the count and the distinct count of gender field of all the accounts. + +PPL query:: + + os> source=accounts | stats count(gender), distinct_count(gender); + fetched rows / total rows = 1/1 + +-----------------+--------------------------+ + | count(gender) | distinct_count(gender) | + |-----------------+--------------------------| + | 4 | 2 | + +-----------------+--------------------------+ + diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index ff3ad2a6c8b..4a9603fe6bd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -77,6 +77,19 @@ public void testStatsCountAll() throws IOException { verifyDataRows(response, rows(1000)); } + @Test + public void testStatsDistinctCount() throws IOException { + JSONObject response = + executeQuery(String.format("source=%s | stats distinct_count(gender)", TEST_INDEX_ACCOUNT)); + verifySchema(response, schema("distinct_count(gender)", null, "integer")); + verifyDataRows(response, rows(2)); + + response = + executeQuery(String.format("source=%s | stats dc(age)", TEST_INDEX_ACCOUNT)); + verifySchema(response, schema("dc(age)", null, "integer")); + verifyDataRows(response, rows(21)); + } + @Test public void testStatsMin() throws IOException { JSONObject response = executeQuery(String.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java index 3cbb222afe1..33cddc6f1f9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java @@ -30,7 +30,15 @@ protected void init() throws Exception { } @Test - void filteredAggregateWithSubquery() throws IOException { + void filteredAggregatePushedDown() throws IOException { + JSONObject response = executeQuery( + "SELECT COUNT(*) FILTER(WHERE age > 35) FROM " + TEST_INDEX_BANK); + verifySchema(response, schema("COUNT(*)", null, "integer")); + verifyDataRows(response, rows(3)); + } + + @Test + void filteredAggregateNotPushedDown() throws IOException { JSONObject response = executeQuery( "SELECT COUNT(*) FILTER(WHERE age > 35) FROM (SELECT * FROM " + TEST_INDEX_BANK + ") AS a"); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java index b92ca17238c..52373a72e32 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java @@ -29,6 +29,8 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; + import org.json.JSONObject; import org.junit.Test; @@ -40,6 +42,7 @@ public class WindowFunctionIT extends SQLIntegTestCase { @Override protected void init() throws Exception { loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.BANK); } @Test @@ -74,4 +77,49 @@ public void testOrderByNullLast() { rows(null, 7)); } + @Test + public void testDistinctCountOverNull() { + JSONObject response = new JSONObject(executeQuery( + "SELECT lastname, COUNT(DISTINCT gender) OVER() " + + "FROM " + TestsConstants.TEST_INDEX_BANK, "jdbc")); + verifyDataRows(response, + rows("Duke Willmington", 2), + rows("Bond", 2), + rows("Bates", 2), + rows("Adams", 2), + rows("Ratliff", 2), + rows("Ayala", 2), + rows("Mcpherson", 2)); + } + + @Test + public void testDistinctCountOver() { + JSONObject response = new JSONObject(executeQuery( + "SELECT lastname, COUNT(DISTINCT gender) OVER(ORDER BY lastname) " + + "FROM " + TestsConstants.TEST_INDEX_BANK, "jdbc")); + verifyDataRowsInOrder(response, + rows("Adams", 1), + rows("Ayala", 2), + rows("Bates", 2), + rows("Bond", 2), + rows("Duke Willmington", 2), + rows("Mcpherson", 2), + rows("Ratliff", 2)); + } + + @Test + public void testDistinctCountPartition() { + JSONObject response = new JSONObject(executeQuery( + "SELECT lastname, COUNT(DISTINCT gender) OVER(PARTITION BY gender ORDER BY lastname) " + + "FROM " + TestsConstants.TEST_INDEX_BANK, "jdbc")); + verifyDataRowsInOrder(response, + rows("Ayala", 1), + rows("Bates", 1), + rows("Mcpherson", 1), + rows("Adams", 1), + rows("Bond", 1), + rows("Duke Willmington", 1), + rows("Ratliff", 1)); + } + } diff --git a/integ-test/src/test/resources/correctness/queries/aggregation.txt b/integ-test/src/test/resources/correctness/queries/aggregation.txt index 45aa658783b..0c0648a9371 100644 --- a/integ-test/src/test/resources/correctness/queries/aggregation.txt +++ b/integ-test/src/test/resources/correctness/queries/aggregation.txt @@ -9,4 +9,6 @@ SELECT MIN(timestamp) FROM opensearch_dashboards_sample_data_flights SELECT VAR_POP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights SELECT VAR_SAMP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights SELECT STDDEV_POP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights -SELECT STDDEV_SAMP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights \ No newline at end of file +SELECT STDDEV_SAMP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights +SELECT COUNT(DISTINCT Origin), COUNT(DISTINCT Dest) FROM opensearch_dashboards_sample_data_flights +SELECT COUNT(DISTINCT Origin) FROM (SELECT * FROM opensearch_dashboards_sample_data_flights) AS flights \ No newline at end of file diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java index 73d58d793ee..cd793c9046f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java @@ -43,11 +43,9 @@ /** * Abstract Aggregation Builder. - * - * @param type of the actual AggregationBuilder to be built. */ @RequiredArgsConstructor -public class AggregationBuilderHelper { +public class AggregationBuilderHelper { private final ExpressionSerializer serializer; @@ -57,7 +55,7 @@ public class AggregationBuilderHelper { * @param expression Expression * @return AggregationBuilder */ - public T build(Expression expression, Function fieldBuilder, + public T build(Expression expression, Function fieldBuilder, Function scriptBuilder) { if (expression instanceof ReferenceExpression) { String fieldName = ((ReferenceExpression) expression).getAttr(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index b1aff2c5b4e..d137cce75d5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -42,11 +42,11 @@ */ public class BucketAggregationBuilder { - private final AggregationBuilderHelper> helper; + private final AggregationBuilderHelper helper; public BucketAggregationBuilder( ExpressionSerializer serializer) { - this.helper = new AggregationBuilderHelper<>(serializer); + this.helper = new AggregationBuilderHelper(serializer); } /** @@ -62,8 +62,8 @@ public List> build( .missingBucket(true) .order(groupPair.getRight()); resultBuilder - .add(helper.build(groupPair.getLeft().getDelegated(), valuesSourceBuilder::field, - valuesSourceBuilder::script)); + .add((CompositeValuesSourceBuilder) helper.build(groupPair.getLeft().getDelegated(), + valuesSourceBuilder::field, valuesSourceBuilder::script)); } return resultBuilder.build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java index 3d402582888..754da498629 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java @@ -32,11 +32,13 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.bucket.filter.FilterAggregationBuilder; +import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStats; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.sql.expression.Expression; @@ -57,11 +59,14 @@ public class MetricAggregationBuilder extends ExpressionNodeVisitor, Object> { - private final AggregationBuilderHelper> helper; + private final AggregationBuilderHelper helper; private final FilterQueryBuilder filterBuilder; + /** + * Constructor. + */ public MetricAggregationBuilder(ExpressionSerializer serializer) { - this.helper = new AggregationBuilderHelper<>(serializer); + this.helper = new AggregationBuilderHelper(serializer); this.filterBuilder = new FilterQueryBuilder(serializer); } @@ -88,9 +93,26 @@ public Pair visitNamedAggregator( NamedAggregator node, Object context) { Expression expression = node.getArguments().get(0); Expression condition = node.getDelegated().condition(); + Boolean distinct = node.getDelegated().distinct(); String name = node.getName(); + String functionName = node.getFunctionName().getFunctionName().toLowerCase(Locale.ROOT); + + if (distinct) { + switch (functionName) { + case "count": + return make( + AggregationBuilders.cardinality(name), + expression, + condition, + name, + new SingleValueParser(name)); + default: + throw new IllegalStateException(String.format( + "unsupported distinct aggregator %s", node.getFunctionName().getFunctionName())); + } + } - switch (node.getFunctionName().getFunctionName()) { + switch (functionName) { case "avg": return make( AggregationBuilders.avg(name), @@ -176,6 +198,24 @@ private Pair make( return Pair.of(aggregationBuilder, parser); } + /** + * Make {@link CardinalityAggregationBuilder} for distinct count aggregations. + */ + private Pair make(CardinalityAggregationBuilder builder, + Expression expression, + Expression condition, + String name, + MetricParser parser) { + CardinalityAggregationBuilder aggregationBuilder = + helper.build(expression, builder::field, builder::script); + if (condition != null) { + return Pair.of( + makeFilterAggregation(aggregationBuilder, condition, name), + FilterParser.builder().name(name).metricsParser(parser).build()); + } + return Pair.of(aggregationBuilder, parser); + } + /** * Replace star or literal with OpenSearch metadata field "_index". Because: 1) Analyzer already * converts * to string literal, literal check here can handle both COUNT(*) and COUNT(1). 2) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 95a23834754..129814d45fb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -32,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.named; import static org.opensearch.sql.expression.DSL.ref; @@ -42,6 +43,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Arrays; +import java.util.Collections; import java.util.List; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; @@ -51,19 +53,21 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.aggregation.AvgAggregator; import org.opensearch.sql.expression.aggregation.CountAggregator; import org.opensearch.sql.expression.aggregation.MaxAggregator; import org.opensearch.sql.expression.aggregation.MinAggregator; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.aggregation.SumAggregator; -import org.opensearch.sql.expression.aggregation.VarianceAggregator; +import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @ExtendWith(MockitoExtension.class) class MetricAggregationBuilderTest { + private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); @Mock private ExpressionSerializer serializer; @@ -258,6 +262,61 @@ void should_build_stddevSamp_aggregation() { stddevSample(Arrays.asList(ref("age", INTEGER)), INTEGER))))); } + @Test + void should_build_cardinality_aggregation() { + assertEquals( + "{\n" + + " \"count(distinct name)\" : {\n" + + " \"cardinality\" : {\n" + + " \"field\" : \"name\"\n" + + " }\n" + + " }\n" + + "}", + buildQuery( + Collections.singletonList(named("count(distinct name)", new CountAggregator( + Collections.singletonList(ref("name", STRING)), INTEGER).distinct(true))))); + } + + @Test + void should_build_filtered_cardinality_aggregation() { + assertEquals( + "{\n" + + " \"count(distinct name) filter(where age > 30)\" : {\n" + + " \"filter\" : {\n" + + " \"range\" : {\n" + + " \"age\" : {\n" + + " \"from\" : 30,\n" + + " \"to\" : null,\n" + + " \"include_lower\" : false,\n" + + " \"include_upper\" : true,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " },\n" + + " \"aggregations\" : {\n" + + " \"count(distinct name) filter(where age > 30)\" : {\n" + + " \"cardinality\" : {\n" + + " \"field\" : \"name\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", + buildQuery(Collections.singletonList(named( + "count(distinct name) filter(where age > 30)", + new CountAggregator(Collections.singletonList(ref("name", STRING)), INTEGER) + .condition(dsl.greater(ref("age", INTEGER), literal(30))) + .distinct(true))))); + } + + @Test + void should_throw_exception_for_unsupported_distinct_aggregator() { + assertThrows(IllegalStateException.class, + () -> buildQuery(Collections.singletonList(named("avg(distinct age)", new AvgAggregator( + Collections.singletonList(ref("name", STRING)), STRING).distinct(true)))), + "unsupported distinct aggregator avg"); + } + @Test void should_throw_exception_for_unsupported_aggregator() { when(aggregator.getFunctionName()).thenReturn(new FunctionName("unsupported_agg")); diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index d552ad0756d..fa3c5b64b77 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -135,6 +135,7 @@ statsAggTerm statsFunction : statsFunctionName LT_PRTHS valueExpression RT_PRTHS #statsFunctionCall | COUNT LT_PRTHS RT_PRTHS #countAllFunctionCall + | (DISTINCT_COUNT | DC) LT_PRTHS valueExpression RT_PRTHS #distinctCountFunctionCall | percentileAggFunction #percentileAggFunctionCall ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 9fdf8d636d5..7da4f90cf0c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -35,6 +35,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CompareExprContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CountAllFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DecimalLiteralContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DistinctCountFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldExpressionContext; @@ -203,6 +204,11 @@ public UnresolvedExpression visitCountAllFunctionCall(CountAllFunctionCallContex return new AggregateFunction("count", AllFields.of()); } + @Override + public UnresolvedExpression visitDistinctCountFunctionCall(DistinctCountFunctionCallContext ctx) { + return new AggregateFunction("count", visit(ctx.valueExpression()), true); + } + @Override public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionContext ctx) { return new AggregateFunction(ctx.PERCENTILE().getText(), visit(ctx.aggField), diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 71ef692abf1..bfb975ba74c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -37,6 +37,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.defaultFieldsArgs; import static org.opensearch.sql.ast.dsl.AstDSL.defaultSortFieldArgs; import static org.opensearch.sql.ast.dsl.AstDSL.defaultStatsArgs; +import static org.opensearch.sql.ast.dsl.AstDSL.distinctAggregate; import static org.opensearch.sql.ast.dsl.AstDSL.doubleLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.equalTo; import static org.opensearch.sql.ast.dsl.AstDSL.eval; @@ -460,6 +461,19 @@ public void testCountFuncCallExpr() { )); } + @Test + public void testDistinctCount() { + assertEqual("source=t | stats distinct_count(a)", + agg( + relation("t"), + exprList( + alias("distinct_count(a)", + distinctAggregate("count", field("a")))), + emptyList(), + emptyList(), + defaultStatsArgs())); + } + @Test public void testEvalFuncCallExpr() { assertEqual("source=t | eval f=abs(a)", diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index fe5526621f3..61d2f5990fa 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -336,8 +336,10 @@ caseFuncAlternative ; aggregateFunction - : functionName=aggregationFunctionName LR_BRACKET functionArg RR_BRACKET #regularAggregateFunctionCall - | COUNT LR_BRACKET STAR RR_BRACKET #countStarFunctionCall + : functionName=aggregationFunctionName LR_BRACKET functionArg RR_BRACKET + #regularAggregateFunctionCall + | COUNT LR_BRACKET STAR RR_BRACKET #countStarFunctionCall + | COUNT LR_BRACKET DISTINCT functionArg RR_BRACKET #distinctCountFunctionCall ; filterClause diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index b1630aed509..8dda63b7505 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -43,6 +43,7 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CountStarFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DataTypeFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DateLiteralContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DistinctCountFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IsNullPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.LikePredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.MathExpressionAtomContext; @@ -171,7 +172,7 @@ public UnresolvedExpression visitShowDescribePattern( public UnresolvedExpression visitFilteredAggregationFunctionCall( OpenSearchSQLParser.FilteredAggregationFunctionCallContext ctx) { AggregateFunction agg = (AggregateFunction) visit(ctx.aggregateFunction()); - return new AggregateFunction(agg.getFuncName(), agg.getField(), visit(ctx.filterClause())); + return agg.condition(visit(ctx.filterClause())); } @Override @@ -212,6 +213,14 @@ public UnresolvedExpression visitRegularAggregateFunctionCall( visitFunctionArg(ctx.functionArg())); } + @Override + public UnresolvedExpression visitDistinctCountFunctionCall(DistinctCountFunctionCallContext ctx) { + return new AggregateFunction( + ctx.COUNT().getText(), + visitFunctionArg(ctx.functionArg()), + true); + } + @Override public UnresolvedExpression visitCountStarFunctionCall(CountStarFunctionCallContext ctx) { return new AggregateFunction("COUNT", AllFields.of()); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java index 1d9516f8162..44c84495c23 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java @@ -36,6 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; +import static org.opensearch.sql.ast.dsl.AstDSL.distinctAggregate; import static org.opensearch.sql.ast.dsl.AstDSL.function; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; @@ -50,6 +51,7 @@ import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.UnresolvedPlan; @@ -167,6 +169,17 @@ void can_build_implicit_group_by_for_aggregator_in_having_clause() { alias("AVG(age)", aggregate("AVG", qualifiedName("age")))))); } + @Test + void can_build_distinct_aggregator() { + assertThat( + buildAggregation("SELECT COUNT(DISTINCT name) FROM test group by age"), + allOf( + hasGroupByItems(alias("age", qualifiedName("age"))), + hasAggregators( + alias("COUNT(DISTINCT name)", distinctAggregate("COUNT", qualifiedName( + "name")))))); + } + @Test void should_build_nothing_if_no_group_by_and_no_aggregators_in_select() { assertNull(buildAggregation("SELECT name FROM test")); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index e4e8028f054..e101eb94049 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -431,6 +431,23 @@ public void canBuildVariance() { buildExprAst("variance(age)")); } + @Test + public void distinctCount() { + assertEquals( + AstDSL.distinctAggregate("count", qualifiedName("name")), + buildExprAst("count(distinct name)") + ); + } + + @Test + public void filteredDistinctCount() { + assertEquals( + AstDSL.filteredDistinctCount("count", qualifiedName("name"), function( + ">", qualifiedName("age"), intLiteral(30))), + buildExprAst("count(distinct name) filter(where age > 30)") + ); + } + private Node buildExprAst(String expr) { OpenSearchSQLLexer lexer = new OpenSearchSQLLexer(new CaseInsensitiveCharStream(expr)); OpenSearchSQLParser parser = new OpenSearchSQLParser(new CommonTokenStream(lexer)); From 21cad567b28a650716d6f6db017e04d0885272ad Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Wed, 4 Aug 2021 08:06:29 -0700 Subject: [PATCH 10/20] Support implicit type conversion from string to temporal (#171) * Support implicit cast from string to temporal types Signed-off-by: Chen Dai * Add comparison test Signed-off-by: Chen Dai * Add doctest Signed-off-by: Chen Dai * Fix doctest Signed-off-by: Chen Dai * Add more user manual Signed-off-by: Chen Dai * Add more user manual Signed-off-by: Chen Dai * Fix doctest Signed-off-by: Chen Dai * Update user manual Signed-off-by: Chen Dai --- .../opensearch/sql/ast/expression/Cast.java | 2 ++ .../sql/data/type/ExprCoreType.java | 8 ++--- .../org/opensearch/sql/expression/DSL.java | 5 ++++ .../function/BuiltinFunctionName.java | 3 +- .../operator/convert/TypeCastOperator.java | 13 ++++++++ .../sql/analysis/ExpressionAnalyzerTest.java | 2 +- .../sql/data/type/ExprTypeTest.java | 9 ++++++ .../function/WideningTypeRuleTest.java | 8 +++++ .../convert/TypeCastOperatorTest.java | 17 +++++++++++ docs/user/general/datatypes.rst | 30 ++++++++++++++----- .../correctness/expressions/cast.txt | 3 ++ 11 files changed, 87 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index bd57d0a8a68..099bd23a582 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -31,6 +31,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_BOOLEAN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_BYTE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DATETIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DOUBLE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; @@ -77,6 +78,7 @@ public class Cast extends UnresolvedExpression { .put("date", CAST_TO_DATE.getName()) .put("time", CAST_TO_TIME.getName()) .put("timestamp", CAST_TO_TIMESTAMP.getName()) + .put("datetime", CAST_TO_DATETIME.getName()) .build(); /** diff --git a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java index 92da09490cd..4fa023bd06d 100644 --- a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java @@ -77,10 +77,10 @@ public enum ExprCoreType implements ExprType { * Date. * Todo. compatible relationship. */ - TIMESTAMP(UNDEFINED), - DATE(UNDEFINED), - TIME(UNDEFINED), - DATETIME(UNDEFINED), + TIMESTAMP(STRING), + DATE(STRING), + TIME(STRING), + DATETIME(STRING), INTERVAL(UNDEFINED), /** diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index f3e69575965..af51d0898ad 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -645,4 +645,9 @@ public FunctionExpression castTimestamp(Expression value) { return (FunctionExpression) repository .compile(BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), Arrays.asList(value)); } + + public FunctionExpression castDatetime(Expression value) { + return (FunctionExpression) repository + .compile(BuiltinFunctionName.CAST_TO_DATETIME.getName(), Arrays.asList(value)); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 0f6feeb94ac..cd668255672 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -186,7 +186,8 @@ public enum BuiltinFunctionName { CAST_TO_BOOLEAN(FunctionName.of("cast_to_boolean")), CAST_TO_DATE(FunctionName.of("cast_to_date")), CAST_TO_TIME(FunctionName.of("cast_to_time")), - CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")); + CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), + CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java index 5f94eb63ee8..c6a84985a09 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java @@ -50,6 +50,7 @@ import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDatetimeValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; @@ -80,6 +81,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); + repository.register(castToDatetime()); } @@ -205,4 +207,15 @@ private static FunctionResolver castToTimestamp() { impl(nullMissingHandling((v) -> v), TIMESTAMP, TIMESTAMP) ); } + + private static FunctionResolver castToDatetime() { + return FunctionDSL.define(BuiltinFunctionName.CAST_TO_DATETIME.getName(), + impl(nullMissingHandling( + (v) -> new ExprDatetimeValue(v.stringValue())), DATETIME, STRING), + impl(nullMissingHandling( + (v) -> new ExprDatetimeValue(v.datetimeValue())), DATETIME, TIMESTAMP), + impl(nullMissingHandling( + (v) -> new ExprDatetimeValue(v.datetimeValue())), DATETIME, DATE) + ); + } } diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index 99a316fd851..b0b1e7e773e 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -160,7 +160,7 @@ public void castAnalyzer() { ); assertThrows(IllegalStateException.class, () -> analyze(AstDSL.cast(AstDSL.unresolvedAttr( - "boolean_value"), AstDSL.stringLiteral("DATETIME")))); + "boolean_value"), AstDSL.stringLiteral("INTERVAL")))); } @Test diff --git a/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java b/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java index 9beb11eb07a..dc63c7d224b 100644 --- a/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java +++ b/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java @@ -34,6 +34,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -41,6 +43,8 @@ import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; @@ -59,7 +63,12 @@ public void isCompatible() { assertTrue(FLOAT.isCompatible(LONG)); assertTrue(FLOAT.isCompatible(INTEGER)); assertTrue(FLOAT.isCompatible(SHORT)); + assertTrue(BOOLEAN.isCompatible(STRING)); + assertTrue(TIMESTAMP.isCompatible(STRING)); + assertTrue(DATE.isCompatible(STRING)); + assertTrue(TIME.isCompatible(STRING)); + assertTrue(DATETIME.isCompatible(STRING)); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java index 9e678c8091b..745aa9cc3b5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/WideningTypeRuleTest.java @@ -30,12 +30,16 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.data.type.WideningTypeRule.IMPOSSIBLE_WIDENING; import static org.opensearch.sql.data.type.WideningTypeRule.TYPE_EQUAL; @@ -73,6 +77,10 @@ class WideningTypeRuleTest { .put(LONG, DOUBLE, 2) .put(FLOAT, DOUBLE, 1) .put(STRING, BOOLEAN, 1) + .put(STRING, TIMESTAMP, 1) + .put(STRING, DATE, 1) + .put(STRING, TIME, 1) + .put(STRING, DATETIME, 1) .put(UNDEFINED, BYTE, 1) .put(UNDEFINED, SHORT, 2) .put(UNDEFINED, INTEGER, 3) diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index cc2acf57102..31bdca14263 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -33,6 +33,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -366,4 +367,20 @@ void castToTimestamp() { assertEquals(TIMESTAMP, expression.type()); assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf(null)); } + + @Test + void castToDatetime() { + FunctionExpression expression = dsl.castDatetime(DSL.literal("2012-08-07 01:01:01")); + assertEquals(DATETIME, expression.type()); + assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf(null)); + + expression = dsl.castDatetime(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); + assertEquals(DATETIME, expression.type()); + assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf(null)); + + expression = dsl.castDatetime(DSL.literal(new ExprDateValue("2012-08-07"))); + assertEquals(DATETIME, expression.type()); + assertEquals(new ExprDatetimeValue("2012-08-07 00:00:00"), expression.valueOf(null)); + } + } diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index fe440728316..c0a3bf62aca 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -147,7 +147,7 @@ The following matrix illustrates the conversions allowed by our query engine for +--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ | TEXT | | | | | | | | | N/A | IE | | | | X | X | X | X | X | X | X | +--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ -| STRING | E | E | E | E | E | E | IE | X | X | N/A | E | E | E | X | X | X | X | X | X | X | +| STRING | E | E | E | E | E | E | IE | X | X | N/A | IE | IE | IE | IE | X | X | X | X | X | X | +--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ | TIMESTAMP | X | X | X | X | X | X | X | X | X | E | N/A | | | X | X | X | X | X | X | X | +--------------+------+-------+---------+------+-------+--------+---------+--------------+------+--------+-----------+------+------+----------+----------+-----------+-----+--------+-----------+---------+ @@ -183,13 +183,14 @@ Here are a few examples for implicit type conversion:: os> SELECT ... 1 = 1.0, - ... 'True' = true; + ... 'True' = true, + ... DATE('2021-06-10') < '2021-06-11'; fetched rows / total rows = 1/1 - +-----------+-----------------+ - | 1 = 1.0 | 'True' = true | - |-----------+-----------------| - | True | True | - +-----------+-----------------+ + +-----------+-----------------+-------------------------------------+ + | 1 = 1.0 | 'True' = true | DATE('2021-06-10') < '2021-06-11' | + |-----------+-----------------+-------------------------------------| + | True | True | True | + +-----------+-----------------+-------------------------------------+ Here are a few examples for explicit type conversion:: @@ -331,6 +332,21 @@ Conversion from TIMESTAMP - Conversion from timestamp is much more straightforward. To convert it to date is to extract the date value, and conversion to time is to extract the time value. Conversion to datetime, it will extracts the datetime value and leave the timezone information over. For example, the result to convert datetime '2020-08-17 14:09:00' UTC to date is date '2020-08-17', to time is '14:09:00' and to datetime is datetime '2020-08-17 14:09:00'. +Conversion from string to date and time types +--------------------------------------------- + +A string can also represent and be converted to date and time types (except to interval type). As long as the string value is of valid format required by the target date and time types, the conversion can happen implicitly or explicitly as follows:: + + os> SELECT + ... TIMESTAMP('2021-06-17 00:00:00') = '2021-06-17 00:00:00', + ... '2021-06-18' < DATE('2021-06-17'), + ... '10:20:00' <= TIME('11:00:00'); + fetched rows / total rows = 1/1 + +------------------------------------------------------------+-------------------------------------+----------------------------------+ + | TIMESTAMP('2021-06-17 00:00:00') = '2021-06-17 00:00:00' | '2021-06-18' < DATE('2021-06-17') | '10:20:00' <= TIME('11:00:00') | + |------------------------------------------------------------+-------------------------------------+----------------------------------| + | True | False | True | + +------------------------------------------------------------+-------------------------------------+----------------------------------+ String Data Types ================= diff --git a/integ-test/src/test/resources/correctness/expressions/cast.txt b/integ-test/src/test/resources/correctness/expressions/cast.txt index 3556e0b7952..2d313203b46 100644 --- a/integ-test/src/test/resources/correctness/expressions/cast.txt +++ b/integ-test/src/test/resources/correctness/expressions/cast.txt @@ -21,3 +21,6 @@ false = 'False' as implicitCast false = 'true' as implicitCast 'TRUE' = true as implicitCast 'false' = true as implicitCast +CAST('2021-06-17 00:00:00' AS TIMESTAMP) = '2021-06-17 00:00:00' as implicitCast +'2021-06-18' < CAST('2021-06-17' AS DATE) as implicitCast +'10:20:00' <= CAST('11:00:00' AS TIME) as implicitCast From fa58ce8860adf387517cdf3874674a20b3c00123 Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 13 Aug 2021 16:24:44 +0000 Subject: [PATCH 11/20] Use externally-defined OpenSearch version when specified. Signed-off-by: dblock --- build.gradle | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 85523b88591..76aaf4099de 100644 --- a/build.gradle +++ b/build.gradle @@ -26,7 +26,7 @@ buildscript { ext { - opensearch_version = "1.0.0" + opensearch_version = System.getProperty("opensearch.version", "1.0.0") } repositories { @@ -55,12 +55,16 @@ repositories { } ext { - opensearchVersion = '1.0.0' + opensearchVersion = System.getProperty("opensearch.version", "1.0.0") isSnapshot = "true" == System.getProperty("build.snapshot", "true") } allprojects { - version = "${opensearchVersion}.0" + version = "${opensearchVersion}" - "-SNAPSHOT" + ".0" + + if (isSnapshot) { + version += "-SNAPSHOT" + } plugins.withId('java') { sourceCompatibility = targetCompatibility = "1.8" From abd97e7363be62d76b63e929fe0b4afbecf2f06a Mon Sep 17 00:00:00 2001 From: "Daniel Doubrovkine (dB.)" Date: Mon, 16 Aug 2021 11:36:53 -0400 Subject: [PATCH 12/20] Use OpenSearch 1.1 and build snapshot by default. (#181) Signed-off-by: dblock --- .github/workflows/sql-test-and-build-workflow.yml | 6 +++--- build.gradle | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 816056bd33d..8d7ffe1edd6 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -21,14 +21,14 @@ jobs: with: repository: 'opensearch-project/OpenSearch' path: OpenSearch - ref: '1.0' + ref: '1.x' - name: Build OpenSearch working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false + run: ./gradlew publishToMavenLocal - name: Build with Gradle - run: ./gradlew build assemble + run: ./gradlew build assemble -Dopensearch.version=1.1.0-SNAPSHOT - name: Create Artifact Path run: | diff --git a/build.gradle b/build.gradle index 76aaf4099de..fce02d546e3 100644 --- a/build.gradle +++ b/build.gradle @@ -26,7 +26,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "1.0.0") + opensearch_version = System.getProperty("opensearch.version", "1.1.0-SNAPSHOT") } repositories { @@ -55,13 +55,11 @@ repositories { } ext { - opensearchVersion = System.getProperty("opensearch.version", "1.0.0") isSnapshot = "true" == System.getProperty("build.snapshot", "true") } allprojects { - version = "${opensearchVersion}" - "-SNAPSHOT" + ".0" - + version = opensearch_version - "-SNAPSHOT" + ".0" if (isSnapshot) { version += "-SNAPSHOT" } From 9ada74383a812baf534af8984a276303e4619574 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Aug 2021 10:13:06 -0700 Subject: [PATCH 13/20] Bump path-parse from 1.0.6 to 1.0.7 in /workbench (#178) Bumps [path-parse](https://github.com/jbgutierrez/path-parse) from 1.0.6 to 1.0.7. - [Release notes](https://github.com/jbgutierrez/path-parse/releases) - [Commits](https://github.com/jbgutierrez/path-parse/commits/v1.0.7) --- updated-dependencies: - dependency-name: path-parse dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- workbench/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workbench/yarn.lock b/workbench/yarn.lock index a342b782cca..7436adc7007 100644 --- a/workbench/yarn.lock +++ b/workbench/yarn.lock @@ -2063,9 +2063,9 @@ path-key@^3.0.0, path-key@^3.1.0: integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== path-parse@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" - integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== path-type@^4.0.0: version "4.0.0" From b6fdd0d64ed4a45dd600d8682d9bb6dd8b9ccd50 Mon Sep 17 00:00:00 2001 From: Joshua Date: Wed, 1 Sep 2021 11:19:55 -0700 Subject: [PATCH 14/20] Workbench: auto dump cypress test data, support security (#199) --- workbench/.cypress/integration/ui.spec.js | 34 +++++++++++++++---- workbench/.cypress/support/commands.js | 41 +++++++++++++++++++++++ workbench/.cypress/support/constants.js | 15 +++++++++ workbench/.cypress/support/index.js | 5 +++ workbench/.cypress/utils/constants.js | 11 ++++++ workbench/cypress.json | 7 +++- 6 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 workbench/.cypress/support/constants.js diff --git a/workbench/.cypress/integration/ui.spec.js b/workbench/.cypress/integration/ui.spec.js index aa32ab18241..abb5ce1bdb3 100644 --- a/workbench/.cypress/integration/ui.spec.js +++ b/workbench/.cypress/integration/ui.spec.js @@ -26,9 +26,32 @@ /// -import { edit } from "brace"; -import { delay, testQueries, verifyDownloadData, files } from "../utils/constants"; +import { edit } from 'brace'; +import { delay, files, testDataSet, testQueries, verifyDownloadData } from '../utils/constants'; +describe('Dump test data', () => { + it('Indexes test data for SQL and PPL', () => { + const dumpDataSet = (url, index) => + cy.request(url).then((response) => { + cy.request({ + method: 'POST', + form: true, + url: 'api/console/proxy', + headers: { + 'content-type': 'application/json;charset=UTF-8', + 'osd-xsrf': true, + }, + qs: { + path: `${index}/_bulk`, + method: 'POST', + }, + body: response.body, + }); + }); + + testDataSet.forEach(({url, index}) => dumpDataSet(url, index)); + }); +}); describe('Test PPL UI', () => { beforeEach(() => { @@ -183,13 +206,12 @@ describe('Test and verify SQL downloads', () => { 'osd-xsrf': true, }, body: { - 'query': 'select * from accounts where balance > 49500' - } + query: 'select * from accounts where balance > 49500', + }, }).then((response) => { if (title === 'Download and verify CSV' || title === 'Download and verify Text') { expect(response.body.data.body).to.have.string(files[file]); - } - else { + } else { expect(response.body.data.resp).to.have.string(files[file]); } }); diff --git a/workbench/.cypress/support/commands.js b/workbench/.cypress/support/commands.js index e4d90a3918c..ab472854d44 100644 --- a/workbench/.cypress/support/commands.js +++ b/workbench/.cypress/support/commands.js @@ -49,3 +49,44 @@ // // -- This will overwrite an existing command -- // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }) + +import { ADMIN_AUTH } from '../utils/constants'; + +Cypress.Commands.overwrite('visit', (originalFn, url, options) => { + // Add the basic auth header when security enabled in the OpenSearch cluster + // https://github.com/cypress-io/cypress/issues/1288 + if (Cypress.env('security_enabled')) { + if (options) { + options.auth = ADMIN_AUTH; + } else { + options = { auth: ADMIN_AUTH }; + } + // Add query parameters - select the default OpenSearch Dashboards tenant + options.qs = { security_tenant: 'private' }; + return originalFn(url, options); + } else { + return originalFn(url, options); + } +}); + +// Be able to add default options to cy.request(), https://github.com/cypress-io/cypress/issues/726 +Cypress.Commands.overwrite('request', (originalFn, ...args) => { + let defaults = {}; + // Add the basic authentication header when security enabled in the OpenSearch cluster + if (Cypress.env('security_enabled')) { + defaults.auth = ADMIN_AUTH; + } + + let options = {}; + if (typeof args[0] === 'object' && args[0] !== null) { + options = Object.assign({}, args[0]); + } else if (args.length === 1) { + [options.url] = args; + } else if (args.length === 2) { + [options.method, options.url] = args; + } else if (args.length === 3) { + [options.method, options.url, options.body] = args; + } + + return originalFn(Object.assign({}, defaults, options)); +}); diff --git a/workbench/.cypress/support/constants.js b/workbench/.cypress/support/constants.js new file mode 100644 index 00000000000..48ee4ff9ffd --- /dev/null +++ b/workbench/.cypress/support/constants.js @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +export const ADMIN_AUTH = { + username: 'admin', + password: 'admin', +}; diff --git a/workbench/.cypress/support/index.js b/workbench/.cypress/support/index.js index a7af4e66b9a..a3ce8c563c1 100644 --- a/workbench/.cypress/support/index.js +++ b/workbench/.cypress/support/index.js @@ -44,3 +44,8 @@ import './commands'; // Alternatively you can use CommonJS syntax: // require('./commands') + +// Switch the base URL of OpenSearch when security enabled in the cluster +if (Cypress.env('security_enabled')) { + Cypress.env('opensearch', 'https://localhost:9200'); +} diff --git a/workbench/.cypress/utils/constants.js b/workbench/.cypress/utils/constants.js index cc5211a444b..b83e7805724 100644 --- a/workbench/.cypress/utils/constants.js +++ b/workbench/.cypress/utils/constants.js @@ -26,6 +26,17 @@ export const delay = 1000; +export const testDataSet = [ + { + url: 'https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/accounts.json', + index: 'accounts', + }, + { + url: 'https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/employee_nested.json', + index: 'employee_nested' + } +] + export const verifyDownloadData = [ { title: 'Download and verify JSON', diff --git a/workbench/cypress.json b/workbench/cypress.json index e5441904ad6..53c4ba96d84 100644 --- a/workbench/cypress.json +++ b/workbench/cypress.json @@ -9,5 +9,10 @@ "videosFolder": ".cypress/videos", "requestTimeout": 60000, "responseTimeout": 60000, - "defaultCommandTimeout": 60000 + "defaultCommandTimeout": 60000, + "env": { + "opensearch": "localhost:9200", + "opensearchDashboards": "localhost:5601", + "security_enabled": true + } } From d68547d585092af1e053d01e1b834259723cd304 Mon Sep 17 00:00:00 2001 From: Joshua Date: Wed, 1 Sep 2021 13:58:38 -0700 Subject: [PATCH 15/20] Workbench: remove curl commands in integtest.sh (#200) Signed-off-by: Joshua Li --- integtest.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/integtest.sh b/integtest.sh index f5822fb49ac..4ed40c3ed4a 100755 --- a/integtest.sh +++ b/integtest.sh @@ -90,8 +90,6 @@ then rm -rf sql cd workbench yarn osd bootstrap - curl -s https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/accounts.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/accounts/_bulk?pretty' --data-binary @- > /dev/null 2>&1 - curl -s https://raw.githubusercontent.com/opensearch-project/sql/main/integ-test/src/test/resources/employee_nested.json | curl -s -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/employee_nested/_bulk?pretty' --data-binary @- > /dev/null 2>&1 npx cypress run else ./gradlew integTest -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain From a274c4b246627c1fdd8ed4387755fb352a70578c Mon Sep 17 00:00:00 2001 From: Joshua Date: Thu, 2 Sep 2021 13:18:50 -0700 Subject: [PATCH 16/20] Fix import path for cypress constant (#201) --- docs/dev/Pagination.md | 2 +- workbench/.cypress/support/commands.js | 2 +- workbench/.cypress/tsconfig.json | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 workbench/.cypress/tsconfig.json diff --git a/docs/dev/Pagination.md b/docs/dev/Pagination.md index 38ab9d17936..4982b13d7f1 100644 --- a/docs/dev/Pagination.md +++ b/docs/dev/Pagination.md @@ -149,7 +149,7 @@ POST _plugins/_sql/close ### 3.3 Support in JDBC -To use the pagination functionality programmatically using the[JDBC 4.1 specification](https://download.oracle.com/otn-pub/jcp/jdbc-4_1-mrel-spec/jdbc4.1-fr-spec.pdf?AuthParam=1574798710_305327d63d91e91e19dd80953454597a), page size is being used as performance hint given by `Statement.setFetchSize()` and “applied to each result set produced by the statement”. We need to re-implement `Statement.executequery()` and `ResultSet.next()` to take advantage of cursor. +To use the pagination functionality programmatically using the[JDBC 4.1 specification](https://download.oracle.com/otn-pub/jcp/jdbc-4_1-mrel-spec/jdbc4.1-fr-spec.pdf), page size is being used as performance hint given by `Statement.setFetchSize()` and “applied to each result set produced by the statement”. We need to re-implement `Statement.executequery()` and `ResultSet.next()` to take advantage of cursor. We will not support backward scroll on result set. The `Statement` must be created with a `ResultSet` type of `ResultSet.TYPE_FORWARD_ONLY`. Attempt to scroll backwards or otherwise jump around in the `ResultSet` should throw an exception. diff --git a/workbench/.cypress/support/commands.js b/workbench/.cypress/support/commands.js index ab472854d44..2525085abb9 100644 --- a/workbench/.cypress/support/commands.js +++ b/workbench/.cypress/support/commands.js @@ -50,7 +50,7 @@ // -- This will overwrite an existing command -- // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }) -import { ADMIN_AUTH } from '../utils/constants'; +const { ADMIN_AUTH } = require('./constants'); Cypress.Commands.overwrite('visit', (originalFn, url, options) => { // Add the basic auth header when security enabled in the OpenSearch cluster diff --git a/workbench/.cypress/tsconfig.json b/workbench/.cypress/tsconfig.json new file mode 100644 index 00000000000..36de33deefd --- /dev/null +++ b/workbench/.cypress/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "allowJs": true, + "baseUrl": "../node_modules", + "types": ["cypress"] + }, + "include": ["**/*.*"] +} From 728d41598cca3557fe274d2a89e671355c1664d9 Mon Sep 17 00:00:00 2001 From: David Cui <53581635+davidcui1225@users.noreply.github.com> Date: Thu, 2 Sep 2021 16:53:09 -0700 Subject: [PATCH 17/20] Bump version to 1.1 for Opensearch 1.1.0.0 release (#202) * bump workbench versions to 1.1 Signed-off-by: David Cui * bump version to 1.1.0.0 for 1.1 release Signed-off-by: David Cui * add release notes for 1.1 Signed-off-by: David Cui * bump odbc files to 1.1.0.0 Signed-off-by: David Cui --- .../draft-release-notes-workflow.yml | 2 +- .../workflows/sql-odbc-release-workflow.yml | 2 +- .../sql-odbc-rename-and-release-workflow.yml | 2 +- .../sql-workbench-test-and-build-workflow.yml | 4 ++-- .../opensearch-sql.release-notes-1.1.0.0.md | 24 +++++++++++++++++++ sql-cli/src/opensearch_sql_cli/__init__.py | 2 +- sql-jdbc/build.gradle | 2 +- sql-odbc/src/CMakeLists.txt | 4 ++-- .../opensearch_sql_odbc/manifest.xml | 2 +- .../opensearch_sql_odbc_dev/manifest.xml | 2 +- workbench/opensearch_dashboards.json | 4 ++-- workbench/package.json | 4 ++-- 12 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 release-notes/opensearch-sql.release-notes-1.1.0.0.md diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index cc68eeab4a0..0c6190bce1f 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 1.0.0.0 + version: 1.1.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index c7901f58c83..6e471248a50 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -12,7 +12,7 @@ env: ODBC_BUILD_PATH: "./build/odbc/build" AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" PLUGIN_NAME: opensearch-sql-odbc - OD_VERSION: 1.0.0.0 + OD_VERSION: 1.1.0.0 jobs: build-mac: diff --git a/.github/workflows/sql-odbc-rename-and-release-workflow.yml b/.github/workflows/sql-odbc-rename-and-release-workflow.yml index 467bf991213..cf79e281ffc 100644 --- a/.github/workflows/sql-odbc-rename-and-release-workflow.yml +++ b/.github/workflows/sql-odbc-rename-and-release-workflow.yml @@ -8,7 +8,7 @@ on: - rename* env: - OD_VERSION: 1.0.0.0 + OD_VERSION: 1.1.0.0 jobs: upload-odbc: diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index cad24e68998..071f7dfb29f 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -4,8 +4,8 @@ on: [pull_request, push] env: PLUGIN_NAME: query-workbench-dashboards - OPENSEARCH_VERSION: '1.0' - OPENSEARCH_PLUGIN_VERSION: 1.0.0.0 + OPENSEARCH_VERSION: '1.x' + OPENSEARCH_PLUGIN_VERSION: 1.1.0.0 jobs: diff --git a/release-notes/opensearch-sql.release-notes-1.1.0.0.md b/release-notes/opensearch-sql.release-notes-1.1.0.0.md new file mode 100644 index 00000000000..473cca7551f --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-1.1.0.0.md @@ -0,0 +1,24 @@ +## 2021-09-02 Version 1.1.0.0 + +Compatible with OpenSearch and OpenSearch Dashboards Version 1.1.0 + +### Enhancements + +* Support implicit type conversion from string to boolean ([#166](https://github.com/opensearch-project/sql/pull/166)) +* Support distinct count aggregation ([#167](https://github.com/opensearch-project/sql/pull/167)) +* Support implicit type conversion from string to temporal ([#171](https://github.com/opensearch-project/sql/pull/171)) +* Workbench: auto dump cypress test data, support security ([#199](https://github.com/opensearch-project/sql/pull/199)) + +### Bug Fixes + +* Fix for SQL-ODBC AWS Init and Shutdown Behaviour ([#163](https://github.com/opensearch-project/sql/pull/163)) +* Fix import path for cypress constant ([#201](https://github.com/opensearch-project/sql/pull/201)) + + +### Infrastructure + +* Add Integtest.sh for OpenSearch integtest setups (workbench) ([#157](https://github.com/opensearch-project/sql/pull/157)) +* Bump path-parse from 1.0.6 to 1.0.7 in /workbench ([#178](https://github.com/opensearch-project/sql/pull/178)) +* Use externally-defined OpenSearch version when specified. ([#179](https://github.com/opensearch-project/sql/pull/179)) +* Use OpenSearch 1.1 and build snapshot by default in CI. ([#181](https://github.com/opensearch-project/sql/pull/181)) +* Workbench: remove curl commands in integtest.sh ([#200](https://github.com/opensearch-project/sql/pull/200)) diff --git a/sql-cli/src/opensearch_sql_cli/__init__.py b/sql-cli/src/opensearch_sql_cli/__init__.py index e9f870deba7..d4f6ce3d731 100644 --- a/sql-cli/src/opensearch_sql_cli/__init__.py +++ b/sql-cli/src/opensearch_sql_cli/__init__.py @@ -22,4 +22,4 @@ express or implied. See the License for the specific language governing permissions and limitations under the License. """ -__version__ = "1.0.0.0" +__version__ = "1.1.0.0" diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 548b34a9be2..516d3f51754 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -43,7 +43,7 @@ plugins { group 'org.opensearch.client' // keep version in sync with version in Driver source -version '1.0.0.0' +version '1.1.0.0' boolean snapshot = "true".equals(System.getProperty("build.snapshot", "false")); if (snapshot) { diff --git a/sql-odbc/src/CMakeLists.txt b/sql-odbc/src/CMakeLists.txt index fd24ea9e8f2..ae5db3b98c3 100644 --- a/sql-odbc/src/CMakeLists.txt +++ b/sql-odbc/src/CMakeLists.txt @@ -87,8 +87,8 @@ set(INSTALL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/installer") set(DSN_INSTALLER_SRC "${CMAKE_CURRENT_SOURCE_DIR}/DSNInstaller") # ODBC Driver version -set(DRIVER_PACKAGE_VERSION "1.0.0.0") -set(DRIVER_PACKAGE_VERSION_COMMA_SEPARATED "1,0,0,0") +set(DRIVER_PACKAGE_VERSION "1.1.0.0") +set(DRIVER_PACKAGE_VERSION_COMMA_SEPARATED "1,1,0,0") add_compile_definitions( OPENSEARCH_ODBC_VERSION="${DRIVER_PACKAGE_VERSION}" # Comma separated version is required for odbc administrator's driver file. OPENSEARCH_ODBC_DRVFILE_VERSION=${DRIVER_PACKAGE_VERSION_COMMA_SEPARATED} ) diff --git a/sql-odbc/src/TableauConnector/opensearch_sql_odbc/manifest.xml b/sql-odbc/src/TableauConnector/opensearch_sql_odbc/manifest.xml index 077e7d01d2a..be2e73897c4 100644 --- a/sql-odbc/src/TableauConnector/opensearch_sql_odbc/manifest.xml +++ b/sql-odbc/src/TableauConnector/opensearch_sql_odbc/manifest.xml @@ -1,6 +1,6 @@ - + diff --git a/sql-odbc/src/TableauConnector/opensearch_sql_odbc_dev/manifest.xml b/sql-odbc/src/TableauConnector/opensearch_sql_odbc_dev/manifest.xml index 0f470f3a8d9..c59a0e74b01 100644 --- a/sql-odbc/src/TableauConnector/opensearch_sql_odbc_dev/manifest.xml +++ b/sql-odbc/src/TableauConnector/opensearch_sql_odbc_dev/manifest.xml @@ -1,6 +1,6 @@ - + diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index 0ee32f9b98a..c1b6ab2677d 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "queryWorkbenchDashboards", - "version": "1.0.0.0", - "opensearchDashboardsVersion": "1.0.0", + "version": "1.1.0.0", + "opensearchDashboardsVersion": "1.1.0", "server": true, "ui": true, "requiredPlugins": ["navigation"], diff --git a/workbench/package.json b/workbench/package.json index 3e78f6a26b2..3d9af584a94 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -1,12 +1,12 @@ { "name": "opensearch-query-workbench", - "version": "1.0.0.0", + "version": "1.1.0.0", "description": "Query Workbench", "main": "index.js", "license": "Apache-2.0", "homepage": "https://github.com/opensearch-project/sql/tree/main/workbench", "opensearchDashboards": { - "version": "1.0.0", + "version": "1.1.0", "templateVersion": "1.0.0" }, "repository": { From 74d9fa01fdf80fafefe746a6e23107fa68e45969 Mon Sep 17 00:00:00 2001 From: Joshua Date: Tue, 14 Sep 2021 12:47:44 -0700 Subject: [PATCH 18/20] Bump opensearch ref to 1.1 in CI (#205) --- .github/workflows/link-checker.yml | 2 +- .../workflows/sql-cli-release-workflow.yml | 2 +- .../sql-cli-test-and-build-workflow.yml | 2 +- .../workflows/sql-jdbc-push-jdbc-maven.yml | 2 +- .github/workflows/sql-release-workflow.yml | 2 +- .../workflows/sql-test-and-build-workflow.yml | 2 +- README.md | 2 +- docs/dev/NewSQLEngine.md | 28 +++++++++---------- .../opensearch-sql.release-notes-1.1.0.0.md | 1 + sql-cli/CONTRIBUTING.md | 4 +-- sql-cli/README.md | 4 +-- sql-jdbc/CONTRIBUTING.md | 2 +- sql-odbc/CONTRIBUTING.md | 2 +- workbench/README.md | 2 +- .../sql-workbench.release-notes-1.7.0.0.md | 2 +- 15 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index be6f481978c..10cab37f8ab 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -16,7 +16,7 @@ jobs: id: lychee uses: lycheeverse/lychee-action@master with: - args: --accept=200,403,429 "**/*.html" "**/*.md" "**/*.txt" --exclude "http://localhost*" "https://localhost" "https://odfe-node1:9200/" "https://community.tableau.com/docs/DOC-17978" ".*family.zzz" "https://pypi.python.org/pypi/opensearch-sql-cli/" "opensearch*" ".*@amazon.com" ".*email.com" "git@github.com" "http://timestamp.verisign.com/scripts/timstamp.dll" + args: --accept=200,403,429,999 "**/*.html" "**/*.md" "**/*.txt" --exclude "http://localhost*" "https://localhost" "https://odfe-node1:9200/" "https://community.tableau.com/docs/DOC-17978" ".*family.zzz" "https://pypi.python.org/pypi/opensearch-sql-cli/" "opensearch*" ".*@amazon.com" ".*email.com" "git@github.com" "http://timestamp.verisign.com/scripts/timstamp.dll" env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - name: Fail if there were link errors diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml index 2959df51b6b..a7042bcd327 100644 --- a/.github/workflows/sql-cli-release-workflow.yml +++ b/.github/workflows/sql-cli-release-workflow.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: [ubuntu-16.04] + runs-on: ubuntu-latest defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index fd83a893735..876780a86c2 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: build: - runs-on: [ubuntu-16.04] + runs-on: ubuntu-latest defaults: run: working-directory: sql-cli diff --git a/.github/workflows/sql-jdbc-push-jdbc-maven.yml b/.github/workflows/sql-jdbc-push-jdbc-maven.yml index 53f2d5d3911..3c964472336 100644 --- a/.github/workflows/sql-jdbc-push-jdbc-maven.yml +++ b/.github/workflows/sql-jdbc-push-jdbc-maven.yml @@ -8,7 +8,7 @@ on: jobs: upload-jdbc-jar: - runs-on: [ubuntu-16.04] + runs-on: ubuntu-latest defaults: run: working-directory: sql-jdbc diff --git a/.github/workflows/sql-release-workflow.yml b/.github/workflows/sql-release-workflow.yml index bbfbd929a84..974f801d368 100644 --- a/.github/workflows/sql-release-workflow.yml +++ b/.github/workflows/sql-release-workflow.yml @@ -12,7 +12,7 @@ jobs: java: [14] name: Build and Release SQL Plugin - runs-on: [ubuntu-16.04] + runs-on: ubuntu-latest steps: - name: Checkout SQL diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 8d7ffe1edd6..5ae3f6aec47 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -21,7 +21,7 @@ jobs: with: repository: 'opensearch-project/OpenSearch' path: OpenSearch - ref: '1.x' + ref: '1.1' - name: Build OpenSearch working-directory: ./OpenSearch diff --git a/README.md b/README.md index 0a663254249..e679c64ccc0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The following projects have been merged into this repository as separate folders Besides basic filtering and aggregation, OpenSearch SQL also supports complex queries, such as querying semi-structured data, JOINs, set operations, sub-queries etc. Beyond the standard functions, OpenSearch functions are provided for better analytics and visualization. Please check our [documentation](#documentation) for more details. -Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported both SQL and Piped Processing Language. Please find more details in [SQL Engine V2 - Release Notes](/docs/dev/NewSQLEngine.md). +Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported both SQL and Piped Processing Language. Please find more details in [SQL Engine V2 - Release Notes](./docs/dev/NewSQLEngine.md). ## Documentation diff --git a/docs/dev/NewSQLEngine.md b/docs/dev/NewSQLEngine.md index 0be3370f33e..cd82e615711 100644 --- a/docs/dev/NewSQLEngine.md +++ b/docs/dev/NewSQLEngine.md @@ -12,23 +12,23 @@ The current SQL query engine provides users the basic query capability for using With the architecture and extensibility improved significantly, the following SQL features are able to be introduced in the new query engine: * **Language Structure** - * [Identifiers](/docs/user/general/identifiers.rst): added support for identifier names with special characters - * [Data types](/docs/user/general/datatypes.rst): added support for date and interval types - * [Expressions](/docs/user/dql/expressions.rst): complex nested expression support - * [SQL functions](/docs/user/dql/functions.rst): more date function support, `ADDDATE`, `DATE_ADD`, `DATE_SUB`, `DAY`, `DAYNAME`, `DAYOFMONTH`, `DAYOFWEEK`, `DAYOFYEAR`, `FROM_DAYS`, `HOUR`, `MICROSECOND`, `MINUTE`, `QUARTER`, `SECOND`, `SUBDATE`, `TIME`, `TIME_TO_SEC`, `TO_DAYS`, `WEEK` - * [Comments](/docs/user/general/comments.rst): SQL comment support + * [Identifiers](../../docs/user/general/identifiers.rst): added support for identifier names with special characters + * [Data types](../../docs/user/general/datatypes.rst): added support for date and interval types + * [Expressions](../../docs/user/dql/expressions.rst): complex nested expression support + * [SQL functions](../../docs/user/dql/functions.rst): more date function support, `ADDDATE`, `DATE_ADD`, `DATE_SUB`, `DAY`, `DAYNAME`, `DAYOFMONTH`, `DAYOFWEEK`, `DAYOFYEAR`, `FROM_DAYS`, `HOUR`, `MICROSECOND`, `MINUTE`, `QUARTER`, `SECOND`, `SUBDATE`, `TIME`, `TIME_TO_SEC`, `TO_DAYS`, `WEEK` + * [Comments](../../docs/user/general/comments.rst): SQL comment support * **Basic queries** - * [HAVING without GROUP BY clause](/docs/user/dql/aggregations.rst#having-without-group-by) - * [Aggregate over arbitrary expression](/docs/user/dql/aggregations.rst#expression) - * [Ordering by NULLS FIRST/LAST](/docs/user/dql/basics.rst#example-2-specifying-order-for-null) - * [Ordering by aggregate function](/docs/user/dql/basics.rst#example-3-ordering-by-aggregate-functions) + * [HAVING without GROUP BY clause](../../docs/user/dql/aggregations.rst#having-without-group-by) + * [Aggregate over arbitrary expression](../../docs/user/dql/aggregations.rst#expression) + * [Ordering by NULLS FIRST/LAST](../../docs/user/dql/basics.rst#example-2-specifying-order-for-null) + * [Ordering by aggregate function](../../docs/user/dql/basics.rst#example-3-ordering-by-aggregate-functions) * **Complex queries** - * [Subqueries in FROM clause](/docs/user/dql/complex.rst#example-2-subquery-in-from-clause): support arbitrary nesting level and aggregation + * [Subqueries in FROM clause](../../docs/user/dql/complex.rst#example-2-subquery-in-from-clause): support arbitrary nesting level and aggregation * **Advanced Features** - * [Window functions](/docs/user/dql/window.rst): ranking and aggregate window functions - * [Selective aggregation](/docs/user/dql/aggregations.rst#filter-clause): by standard `FILTER` function + * [Window functions](../../docs/user/dql/window.rst): ranking and aggregate window functions + * [Selective aggregation](../../docs/user/dql/aggregations.rst#filter-clause): by standard `FILTER` function * **Beyond SQL** - * [Semi-structured data query](/docs/user/beyond/partiql.rst#example-2-selecting-deeper-levels): support querying OpenSearch object fields on arbitrary level + * [Semi-structured data query](../../docs/user/beyond/partiql.rst#example-2-selecting-deeper-levels): support querying OpenSearch object fields on arbitrary level * OpenSearch multi-field: handled automatically and users won't have the access, ex. `text` is converted to `text.keyword` if it’s a multi-field As for correctness, besides full coverage of unit and integration test, we developed a new comparison test framework to ensure correctness by comparing with other databases. Please find more details in [Testing](./Testing.md). @@ -57,7 +57,7 @@ For the following features unsupported in the new engine, the query will be forw ### 3.3 Limitations -You can find all the limitations in [Limitations](/docs/user/limitations/limitations.rst). +You can find all the limitations in [Limitations](../../docs/user/limitations/limitations.rst). --- diff --git a/release-notes/opensearch-sql.release-notes-1.1.0.0.md b/release-notes/opensearch-sql.release-notes-1.1.0.0.md index 473cca7551f..a99aa7b56a4 100644 --- a/release-notes/opensearch-sql.release-notes-1.1.0.0.md +++ b/release-notes/opensearch-sql.release-notes-1.1.0.0.md @@ -22,3 +22,4 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 1.1.0 * Use externally-defined OpenSearch version when specified. ([#179](https://github.com/opensearch-project/sql/pull/179)) * Use OpenSearch 1.1 and build snapshot by default in CI. ([#181](https://github.com/opensearch-project/sql/pull/181)) * Workbench: remove curl commands in integtest.sh ([#200](https://github.com/opensearch-project/sql/pull/200)) +* Bump opensearch ref to 1.1 in CI ([#205](https://github.com/opensearch-project/dashboards-reports/pull/205)) diff --git a/sql-cli/CONTRIBUTING.md b/sql-cli/CONTRIBUTING.md index a0c928b87c0..231b2be86a7 100644 --- a/sql-cli/CONTRIBUTING.md +++ b/sql-cli/CONTRIBUTING.md @@ -58,11 +58,11 @@ If you've thought of a way that OpenSearch could be better, we want to hear abou As with other types of contributions, the first step is to [**open an issue on GitHub**](https://github.com/opensearch-project/OpenSearch/issues/new/choose). Opening an issue before you make changes makes sure that someone else isn't already working on that particular problem. It also lets us all work together to find the right approach before you spend a bunch of time on a PR. So again, when in doubt, open an issue. -Once you've opened an issue, check out our [Developer Guide](./DEVELOPER_GUIDE.md) for instructions on how to get started. +Once you've opened an issue, check out our [Developer Guide](../DEVELOPER_GUIDE.rst) for instructions on how to get started. ## Developer Certificate of Origin -OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](./LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. +OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](../LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. We respect intellectual property rights of others and we want to make sure all incoming contributions are correctly attributed and licensed. A Developer Certificate of Origin (DCO) is a lightweight mechanism to do that. diff --git a/sql-cli/README.md b/sql-cli/README.md index 89f8ba59779..3e690c2a9cd 100644 --- a/sql-cli/README.md +++ b/sql-cli/README.md @@ -123,7 +123,7 @@ Run single query from command line with options ## Code of Conduct -This project has adopted an [Open Source Code of Conduct](/CODE_OF_CONDUCT.md). +This project has adopted an [Open Source Code of Conduct](./CODE_OF_CONDUCT.md). @@ -133,7 +133,7 @@ If you discover a potential security issue in this project we ask that you notif ## Licensing -See the [LICENSE](/LICENSE.TXT) file for our project's licensing. We will ask you to confirm the licensing of your contribution. +See the [LICENSE](./LICENSE.TXT) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/sql-jdbc/CONTRIBUTING.md b/sql-jdbc/CONTRIBUTING.md index 1461eb76c9f..4fb91b52836 100644 --- a/sql-jdbc/CONTRIBUTING.md +++ b/sql-jdbc/CONTRIBUTING.md @@ -20,7 +20,7 @@ reported the issue. Please try to include as much information as you can. Detail * Anything unusual about your environment or deployment ## Sign your work -OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](./LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. +OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](../LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. We respect intellectual property rights of others and we want to make sure all incoming contributions are correctly attributed and licensed. A Developer Certificate of Origin (DCO) is a lightweight mechanism to do that. diff --git a/sql-odbc/CONTRIBUTING.md b/sql-odbc/CONTRIBUTING.md index bd03ee53a7b..c83884fa503 100644 --- a/sql-odbc/CONTRIBUTING.md +++ b/sql-odbc/CONTRIBUTING.md @@ -20,7 +20,7 @@ reported the issue. Please try to include as much information as you can. Detail * Anything unusual about your environment or deployment ## Sign your work -OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](./LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. +OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](../LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. We respect intellectual property rights of others and we want to make sure all incoming contributions are correctly attributed and licensed. A Developer Certificate of Origin (DCO) is a lightweight mechanism to do that. diff --git a/workbench/README.md b/workbench/README.md index e8c50e6b589..de371a84791 100644 --- a/workbench/README.md +++ b/workbench/README.md @@ -28,7 +28,7 @@ If you discover a potential security issue in this project we ask that you notif ## License -This project is licensed under the [Apache v2.0 License](LICENSE.txt). +This project is licensed under the [Apache v2.0 License](../LICENSE.txt). ## Copyright diff --git a/workbench/release-notes/sql-workbench.release-notes-1.7.0.0.md b/workbench/release-notes/sql-workbench.release-notes-1.7.0.0.md index c9976a86009..34c0ae444e5 100644 --- a/workbench/release-notes/sql-workbench.release-notes-1.7.0.0.md +++ b/workbench/release-notes/sql-workbench.release-notes-1.7.0.0.md @@ -18,7 +18,7 @@ To use the SQL Workbench, you will need the [Open Distro SQL plugin](https://git - Updated configureation for v7.3 compatibility ([#7](https://github.com/opendistro-for-elasticsearch/sql-workbench/pull/7)) - Opendistro-1.3 compatible with ES and Kibana 7.3 ([#8](https://github.com/opendistro-for-elasticsearch/sql-workbench/pull/8)) - Changed kibana version to v7.3.2 ([#9](https://github.com/opendistro-for-elasticsearch/sql-workbench/pull/9)) -- Support v7.1.1 Compatibility ([#13](v13)) +- Support v7.1.1 Compatibility ([#13](https://github.com/opendistro-for-elasticsearch/sql-workbench/pull/13)) - Bump pr-branch to v1.3 ([#21](https://github.com/opendistro-for-elasticsearch/sql-workbench/pull/21)) - Support v7.4 compatibility for kibana and sql-plugin ([#23](https://github.com/opendistro-for-elasticsearch/sql-workbench/pull/23)) - Improve the performance by ridding of sending redundant requests ([#24](https://github.com/opendistro-for-elasticsearch/sql-workbench/pull/24)) From 7cda06b7675cc074ceb3d12e30fd44a01a68b5b5 Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Thu, 16 Sep 2021 11:15:43 -0700 Subject: [PATCH 19/20] Fix PPL request concurrency handling issue (#207) * Downscope request to local method Signed-off-by: Chen Dai * Fix checkstyle Signed-off-by: Chen Dai --- .../opensearch/sql/plugin/rest/RestPPLQueryAction.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 4a5f35bddbf..5a230b8d055 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -97,8 +97,6 @@ public class RestPPLQueryAction extends BaseRestHandler { private final Supplier pplEnabled; - private PPLQueryRequest pplRequest; - /** * Constructor of RestPPLQueryAction. */ @@ -155,12 +153,12 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod } PPLService pplService = createPPLService(nodeClient); - pplRequest = PPLQueryRequestFactory.getPPLRequest(request); + PPLQueryRequest pplRequest = PPLQueryRequestFactory.getPPLRequest(request); if (pplRequest.isExplainRequest()) { return channel -> pplService.explain(pplRequest, createExplainResponseListener(channel)); } - return channel -> pplService.execute(pplRequest, createListener(channel)); + return channel -> pplService.execute(pplRequest, createListener(channel, pplRequest)); } /** @@ -215,7 +213,8 @@ public void onFailure(Exception e) { }; } - private ResponseListener createListener(RestChannel channel) { + private ResponseListener createListener(RestChannel channel, + PPLQueryRequest pplRequest) { Format format = pplRequest.format(); ResponseFormatter formatter; if (format.equals(Format.CSV)) { From 012cc0321a08a06486f048ad65deda2a72144d5d Mon Sep 17 00:00:00 2001 From: "Daniel Doubrovkine (dB.)" Date: Thu, 16 Sep 2021 17:27:55 -0400 Subject: [PATCH 20/20] Removed integtest.sh. (#208) Signed-off-by: dblock --- integtest.sh | 97 ---------------------------------------------------- 1 file changed, 97 deletions(-) delete mode 100755 integtest.sh diff --git a/integtest.sh b/integtest.sh deleted file mode 100755 index 4ed40c3ed4a..00000000000 --- a/integtest.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -set -e - -function usage() { - echo "" - echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster." - echo "--------------------------------------------------------------------------" - echo "Usage: $0 [args]" - echo "" - echo "Required arguments:" - echo "None" - echo "" - echo "Optional arguments:" - echo -e "-b BIND_ADDRESS\t, defaults to localhost | 127.0.0.1, can be changed to any IP or domain name for the cluster location." - echo -e "-p BIND_PORT\t, defaults to 9200 or 5601 depends on OpenSearch or Dashboards, can be changed to any port for the cluster location." - echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." - echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." - echo -e "-d DIRECTORY\tSome Repo has more than 1 directory / component, use this to give that directory for separate tests." - echo -e "-h\tPrint this message." - echo "--------------------------------------------------------------------------" -} - -while getopts ":hb:p:s:c:d:" arg; do - case $arg in - h) - usage - exit 1 - ;; - b) - BIND_ADDRESS=$OPTARG - ;; - p) - BIND_PORT=$OPTARG - ;; - s) - SECURITY_ENABLED=$OPTARG - ;; - c) - CREDENTIAL=$OPTARG - ;; - d) - DIRECTORY=$OPTARG - ;; - :) - echo "-${OPTARG} requires an argument" - usage - exit 1 - ;; - ?) - echo "Invalid option: -${OPTARG}" - exit 1 - ;; - esac -done - - -if [ -z "$BIND_ADDRESS" ] -then - BIND_ADDRESS="localhost" -fi - -if [ -z "$BIND_PORT" ] -then - BIND_PORT="9200" -fi - -if [ -z "$SECURITY_ENABLED" ] -then - SECURITY_ENABLED="true" -fi - -if [ -z "$CREDENTIAL" ] -then - CREDENTIAL="admin:admin" - USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` - PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` -fi - -if [ -z "$DIRECTORY" ] -then - DIRECTORY="root" -fi - - -if [ "$DIRECTORY" = "workbench" ] -then - mv -v workbench ../ - cd ../ - rm -rf sql - cd workbench - yarn osd bootstrap - npx cypress run -else - ./gradlew integTest -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain - -fi